From 83c8415fecb210a0dd82cabe59559a0e61e52111 Mon Sep 17 00:00:00 2001 From: rimu <3310831+rimu@users.noreply.github.com> Date: Thu, 10 Aug 2023 21:13:37 +1200 Subject: [PATCH] activitypub - actors and their outboxes --- app/activitypub/routes.py | 170 +++++++++++++++++++++++++++----- app/activitypub/util.py | 124 +++++++++++++++++++++++ app/constants.py | 6 ++ app/main/routes.py | 4 +- app/models.py | 23 ++++- app/templates/user_profile.html | 0 requirements.txt | 1 + 7 files changed, 304 insertions(+), 24 deletions(-) create mode 100644 app/constants.py create mode 100644 app/templates/user_profile.html diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index 79d0f0da..784eac23 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -4,7 +4,8 @@ from app import db from app.activitypub import bp from flask import request, Response, render_template, current_app, abort, jsonify from app.models import User, Community -from app.activitypub.util import public_key +from app.activitypub.util import public_key, users_total, active_half_year, active_month, local_posts, local_comments, \ + post_to_activity INBOX = [] @@ -65,11 +66,6 @@ def nodeinfo(): @bp.route('/nodeinfo/2.0') def nodeinfo2(): - users_total = db.session.execute(text('SELECT COUNT(id) as c FROM "user" WHERE ap_id is null AND verified is true AND banned is false AND deleted is false')).scalar() - active_half_year = db.session.execute(text("SELECT COUNT(id) as c FROM \"user\" WHERE last_seen >= CURRENT_DATE - INTERVAL '6 months' AND ap_id is null AND verified is true AND banned is false AND deleted is false")).scalar() - active_month = db.session.execute(text("SELECT COUNT(id) as c FROM \"user\" WHERE last_seen >= CURRENT_DATE - INTERVAL '1 month' AND ap_id is null AND verified is true AND banned is false AND deleted is false")).scalar() - local_posts = db.session.execute(text('SELECT COUNT(id) as c FROM "post" WHERE ap_id is null')).scalar() - local_comments = db.session.execute(text('SELECT COUNT(id) as c FROM "post_reply" WHERE ap_id is null')).scalar() nodeinfo_data = { "version": "2.0", @@ -82,30 +78,160 @@ def nodeinfo2(): ], "usage": { "users": { - "total": users_total, - "activeHalfyear": active_half_year, - "activeMonth": active_month + "total": users_total(), + "activeHalfyear": active_half_year(), + "activeMonth": active_month() }, - "localPosts": local_posts, - "localComments": local_comments + "localPosts": local_posts(), + "localComments": local_comments() }, "openRegistrations": True } return jsonify(nodeinfo_data) -@bp.route('/users/', methods=['GET']) -def return_actor(actor): - """ This returns the actor.json object when somebody in the - Fediverse searches for this user. It returns the paths of - this user's inbox, its preferred username, the user's public key - and a profile image """ +@bp.route('/u/', methods=['GET']) +def user_profile(actor): + """ Requests to this endpoint can be for a JSON representation of the user, or a HTML rendering of their profile. + The two types of requests are differentiated by the header """ + actor = actor.strip() + user = User.query.filter_by(user_name=actor, deleted=False, banned=False, ap_id=None).first() + if user is not None: + if 'application/ld+json' in request.headers.get('Accept', '') or request.accept_mimetypes.accept_json: + server = current_app.config['SERVER_NAME'] + actor_data = { "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + { + "manuallyApprovesFollowers": "as:manuallyApprovesFollowers", + "schema": "http://schema.org#", + "PropertyValue": "schema:PropertyValue", + "value": "schema:value" + } + ], + "type": "Person", + "id": f"https://{server}/u/{actor}", + "preferredUsername": actor, + "inbox": f"https://{server}/u/{actor}/inbox", + "outbox": f"https://{server}/u/{actor}/outbox", + "publicKey": { + "id": f"https://{server}/u/{actor}#main-key", + "owner": f"https://{server}/u/{actor}", + "publicKeyPem": user.public_key.replace("\n", "\\n") + }, + "endpoints": { + "sharedInbox": f"https://{server}/inbox" + }, + "published": user.created.isoformat() + } + if user.avatar_id is not None: + actor_data["icon"] = { + "type": "Image", + "url": f"https://{server}/avatars/{user.avatar.file_path}" + } + resp = jsonify(actor_data) + resp.content_type = 'application/activity+json' + return resp + else: + return render_template('user_profile.html', user=user) - preferredUsername = actor # but could become a custom username, set by the user, stored in the database this preferredUsername doesn't show up yet in Mastodon ... - json = render_template('actor.json', actor=actor, preferredUsername=preferredUsername, publicKey=public_key(), - domain=current_app.config['SERVER_NAME']) # actor = alice - resp = Response(json, status=200, mimetype='application/json') - return resp + +@bp.route('/c/', methods=['GET']) +def community_profile(actor): + """ Requests to this endpoint can be for a JSON representation of the community, or a HTML rendering of it. + The two types of requests are differentiated by the header """ + actor = actor.strip() + community = Community.query.filter_by(name=actor, banned=False, ap_id=None).first() + if community is not None: + if 'application/ld+json' in request.headers.get('Accept', '') or request.accept_mimetypes.accept_json: + server = current_app.config['SERVER_NAME'] + actor_data = {"@context": [ + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + { + "manuallyApprovesFollowers": "as:manuallyApprovesFollowers", + "schema": "http://schema.org#", + "PropertyValue": "schema:PropertyValue", + "value": "schema:value" + } + ], + "type": "Group", + "id": f"https://{server}/c/{actor}", + "name": actor.title, + "summary": actor.description, + "sensitive": True if actor.nsfw or actor.nsfl else False, + "preferredUsername": actor, + "inbox": f"https://{server}/c/{actor}/inbox", + "outbox": f"https://{server}/c/{actor}/outbox", + "followers": f"https://{server}/c/{actor}/followers", + "moderators": f"https://{server}/c/{actor}/moderators", + "featured": f"https://{server}/c/{actor}/featured", + "attributedTo": f"https://{server}/c/{actor}/moderators", + "postingRestrictedToMods": actor.restricted_to_mods, + "url": f"https://{server}/c/{actor}", + "publicKey": { + "id": f"https://{server}/c/{actor}#main-key", + "owner": f"https://{server}/c/{actor}", + "publicKeyPem": community.public_key.replace("\n", "\\n") + }, + "endpoints": { + "sharedInbox": f"https://{server}/inbox" + }, + "published": community.created.isoformat(), + "updated": community.last_active.isoformat(), + } + if community.avatar_id is not None: + actor_data["icon"] = { + "type": "Image", + "url": f"https://{server}/avatars/{community.avatar.file_path}" + } + resp = jsonify(actor_data) + resp.content_type = 'application/activity+json' + return resp + else: + return render_template('user_profile.html', user=community) + + +@bp.route('/c//outbox', methods=['GET']) +def community_outbox(actor): + actor = actor.strip() + community = Community.query.filter_by(name=actor, banned=False, ap_id=None).first() + if community is not None: + posts = community.posts.limit(50).all() + + community_data = { + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + { + "lemmy": "https://join-lemmy.org/ns#", + "pt": "https://joinpeertube.org/ns#", + "sc": "http://schema.org/", + "commentsEnabled": "pt:commentsEnabled", + "sensitive": "as:sensitive", + "postingRestrictedToMods": "lemmy:postingRestrictedToMods", + "removeData": "lemmy:removeData", + "stickied": "lemmy:stickied", + "moderators": { + "@type": "@id", + "@id": "lemmy:moderators" + }, + "expires": "as:endTime", + "distinguished": "lemmy:distinguished", + "language": "sc:inLanguage", + "identifier": "sc:identifier" + } + ], + "type": "OrderedCollection", + "id": f"https://{current_app.config['SERVER_NAME']}/c/{actor}/outbox", + "totalItems": len(posts), + "orderedItems": [] + } + + for post in posts: + community_data['orderedItems'].append(post_to_activity(post, community)) + + return jsonify(community_data) @bp.route('/inspect') diff --git a/app/activitypub/util.py b/app/activitypub/util.py index 9e6833eb..99e2fe54 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -1,4 +1,14 @@ import os +from flask import current_app +from sqlalchemy import text +from app import db +from app.models import User, Post, Community +import time +import base64 +import requests +from cryptography.hazmat.primitives import serialization, hashes +from cryptography.hazmat.primitives.asymmetric import padding +from app.constants import * def public_key(): @@ -10,3 +20,117 @@ def public_key(): PUBLICKEY = publicKey.replace('\n', '\\n') # JSON-LD doesn't want to work with linebreaks, # but needs the \n character to know where to break the line ;) return PUBLICKEY + + +def users_total(): + return db.session.execute(text( + 'SELECT COUNT(id) as c FROM "user" WHERE ap_id is null AND verified is true AND banned is false AND deleted is false')).scalar() + + +def active_half_year(): + return db.session.execute(text( + "SELECT COUNT(id) as c FROM \"user\" WHERE last_seen >= CURRENT_DATE - INTERVAL '6 months' AND ap_id is null AND verified is true AND banned is false AND deleted is false")).scalar() + + +def active_month(): + return db.session.execute(text( + "SELECT COUNT(id) as c FROM \"user\" WHERE last_seen >= CURRENT_DATE - INTERVAL '1 month' AND ap_id is null AND verified is true AND banned is false AND deleted is false")).scalar() + + +def local_posts(): + return db.session.execute(text('SELECT COUNT(id) as c FROM "post" WHERE ap_id is null')).scalar() + + +def local_comments(): + return db.session.execute(text('SELECT COUNT(id) as c FROM "post_reply" WHERE ap_id is null')).scalar() + + +def send_activity(sender: User, host: str, content: str): + + date = time.strftime('%a, %d %b %Y %H:%M:%S UTC', time.gmtime()) + + private_key = serialization.load_pem_private_key(sender.private_key, password=None) + + # todo: look up instance details to set host_inbox + host_inbox = '/inbox' + + signed_string = f"(request-target): post {host_inbox}\nhost: {host}\ndate: " + date + signature = private_key.sign(signed_string.encode('utf-8'), padding.PKCS1v15(), hashes.SHA256()) + encoded_signature = base64.b64encode(signature).decode('utf-8') + + # Construct the Signature header + header = f'keyId="https://{current_app.config["SERVER_NAME"]}/u/{sender.user_name}",headers="(request-target) host date",signature="{encoded_signature}"' + + # Create headers for the request + headers = { + 'Host': host, + 'Date': date, + 'Signature': header + } + + # Make the HTTP request + try: + response = requests.post(f'https://{host}{host_inbox}', headers=headers, data=content, + timeout=REQUEST_TIMEOUT) + except requests.exceptions.RequestException: + time.sleep(1) + response = requests.post(f'https://{host}{host_inbox}', headers=headers, data=content, + timeout=REQUEST_TIMEOUT / 2) + return response.status_code + + +def post_to_activity(post: Post, community: Community): + activity_data = { + "actor": f"https://{current_app.config['SERVER_NAME']}/c/{community.name}", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "object": { + "id": f"https://{current_app.config['SERVER_NAME']}/activities/create/{post.ap_create_id}", + "actor": f"https://{current_app.config['SERVER_NAME']}/u/{post.author.user_name}", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "object": { + "type": "Page", + "id": f"https://{current_app.config['SERVER_NAME']}/post/{post.id}", + "attributedTo": f"https://{current_app.config['SERVER_NAME']}/u/{post.author.user_name}", + "to": [ + f"https://{current_app.config['SERVER_NAME']}/c/{community.name}", + "https://www.w3.org/ns/activitystreams#Public" + ], + "name": post.title, + "cc": [], + "content": post.body_html, + "mediaType": "text/html", + "source": { + "content": post.body, + "mediaType": "text/markdown" + }, + "attachment": [], + "commentsEnabled": True, + "sensitive": post.nsfw or post.nsfl, + "published": post.created_at, + "audience": f"https://{current_app.config['SERVER_NAME']}/c/{community.name}" + }, + "cc": [ + f"https://{current_app.config['SERVER_NAME']}/c/{community.name}" + ], + "type": "Create", + "audience": f"https://{current_app.config['SERVER_NAME']}/c/{community.name}" + }, + "cc": [ + f"https://{current_app.config['SERVER_NAME']}/c/{community.name}/followers" + ], + "type": "Announce", + "id": f"https://{current_app.config['SERVER_NAME']}/activities/announce/{post.ap_announce_id}" + } + if post.edited_at is not None: + activity_data["object"]["object"]["updated"] = post.edited_at + if post.language is not None: + activity_data["object"]["object"]["language"] = {"identifier": post.language} + if post.type == POST_TYPE_LINK and post.url is not None: + activity_data["object"]["object"]["attachment"] = {"href": post.url, "type": "Link"} + if post.image_id is not None: + activity_data["object"]["object"]["image"] = {"href": post.image.source_url, "type": "Image"} + return activity_data diff --git a/app/constants.py b/app/constants.py new file mode 100644 index 00000000..22c69791 --- /dev/null +++ b/app/constants.py @@ -0,0 +1,6 @@ +REQUEST_TIMEOUT = 2 + +POST_TYPE_LINK = 1 +POST_TYPE_ARTICLE = 2 +POST_TYPE_IMAGE = 3 +POST_TYPE_VIDEO = 4 diff --git a/app/main/routes.py b/app/main/routes.py index e20658ca..8f51e1aa 100644 --- a/app/main/routes.py +++ b/app/main/routes.py @@ -1,5 +1,7 @@ +from datetime import datetime + from app.main import bp -from flask import g +from flask import g, jsonify from flask_moment import moment from flask_babel import _, get_locale diff --git a/app/models.py b/app/models.py index beb1d509..3ad3054c 100644 --- a/app/models.py +++ b/app/models.py @@ -50,10 +50,14 @@ class Community(db.Model): ap_domain = db.Column(db.String(255)) banned = db.Column(db.Boolean, default=False) + restricted_to_mods = db.Column(db.Boolean, default=False) searchable = db.Column(db.Boolean, default=True) search_vector = db.Column(TSVectorType('name', 'title', 'description')) + posts = db.relationship('Post', backref='community', lazy='dynamic', cascade="all, delete-orphan") + replies = db.relationship('PostReply', backref='community', lazy='dynamic', cascade="all, delete-orphan") + class User(UserMixin, db.Model): id = db.Column(db.Integer, primary_key=True) @@ -94,6 +98,9 @@ class User(UserMixin, db.Model): search_vector = db.Column(TSVectorType('user_name', 'bio', 'keywords')) activity = db.relationship('ActivityLog', backref='account', lazy='dynamic', cascade="all, delete-orphan") + avatar = db.relationship(File, foreign_keys=[avatar_id], cascade="all, delete-orphan") + posts = db.relationship('Post', backref='author', lazy='dynamic', cascade="all, delete-orphan") + post_replies = db.relationship('PostReply', backref='author', lazy='dynamic', cascade="all, delete-orphan") def __repr__(self): return ''.format(self.user_name) @@ -166,6 +173,7 @@ class Post(db.Model): title = db.Column(db.String(255)) url = db.Column(db.String(2048)) body = db.Column(db.Text) + body_html = db.Column(db.Text) type = db.Column(db.Integer) has_embed = db.Column(db.Boolean, default=False) reply_count = db.Column(db.Integer, default=0) @@ -184,9 +192,13 @@ class Post(db.Model): edited_at = db.Column(db.DateTime) ap_id = db.Column(db.String(255), index=True) + ap_create_id = db.Column(db.String(100)) + ap_announce_id = db.Column(db.String(100)) search_vector = db.Column(TSVectorType('title', 'body')) + image = db.relationship(File, foreign_keys=[image_id], cascade="all, delete") + class PostReply(db.Model): id = db.Column(db.Integer, primary_key=True) @@ -197,6 +209,7 @@ class PostReply(db.Model): parent_id = db.Column(db.Integer) root_id = db.Column(db.Integer) body = db.Column(db.Text) + body_html = db.Column(db.Text) score = db.Column(db.Integer, default=0, index=True) nsfw = db.Column(db.Boolean, default=False) nsfl = db.Column(db.Boolean, default=False) @@ -258,12 +271,20 @@ class UserBlock(db.Model): class BannedInstances(db.Model): id = db.Column(db.Integer, primary_key=True) - domain = db.Column(db.String(256)) + domain = db.Column(db.String(256), index=True) reason = db.Column(db.String(256)) initiator = db.Column(db.String(256)) created_at = db.Column(db.DateTime, default=datetime.utcnow) +class Instance(db.Model): + id = db.Column(db.Integer, primary_key=True) + domain = db.Column(db.String(256)) + inbox = db.Column(db.String(256)) + shared_inbox = db.Column(db.String(256)) + outbox = db.Column(db.String(256)) + + class Settings(db.Model): name = db.Column(db.String(50), primary_key=True) value = db.Column(db.String(1024)) diff --git a/app/templates/user_profile.html b/app/templates/user_profile.html new file mode 100644 index 00000000..e69de29b diff --git a/requirements.txt b/requirements.txt index b31dea58..b1809abd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,3 +13,4 @@ requests==2.31.0 pyjwt==2.8.0 SQLAlchemy-Searchable==1.4.1 SQLAlchemy-Utils==0.41.1 +cryptography==41.0.3