From 20dfc5a43b731f9ce5452018d9aeab2e4abef811 Mon Sep 17 00:00:00 2001 From: rimu <3310831+rimu@users.noreply.github.com> Date: Fri, 8 Sep 2023 20:04:01 +1200 Subject: [PATCH] follow remote communities - activitypub --- CONTRIBUTING.md | 2 +- README.md | 8 -- app/activitypub/routes.py | 63 +++++---- app/activitypub/util.py | 132 ++++++++++++++++-- app/community/routes.py | 50 +++++-- app/constants.py | 2 + app/models.py | 38 ++++- app/templates/community/community.html | 2 +- .../versions/251be00ae302_follow_requests.py | 50 +++++++ .../versions/cc98a471a1ad_community_ban.py | 39 ++++++ requirements.txt | 1 + 11 files changed, 326 insertions(+), 61 deletions(-) create mode 100644 migrations/versions/251be00ae302_follow_requests.py create mode 100644 migrations/versions/cc98a471a1ad_community_ban.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 517b150c..6fba7d2d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ # Contributing to PyFedi Please discuss your ideas in an issue at https://codeberg.org/rimu/pyfedi/issues before -starting any work to ensure alignment with the roadmap, architecture and processes. +starting any large pieces of work to ensure alignment with the roadmap, architecture and processes. Mailing list, Matrix channel, etc still to come. \ No newline at end of file diff --git a/README.md b/README.md index 3f4552d8..a0f11278 100644 --- a/README.md +++ b/README.md @@ -6,11 +6,3 @@ A lemmy/kbin clone written in Python with Flask. - Easy setup, easy to manage - few dependencies and extra software required. - GPL. -## Contributing - -Join our Maxtrix channel. - -Report bugs in our issue queue. - -If submitting a substantial PR, communicate your intentions to the maintainer(s) before investing your effort into coding. -This is to ensure your contribution will fit well with the rest of the code base. \ No newline at end of file diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index 824fce1d..56813f9d 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -2,12 +2,13 @@ from sqlalchemy import text from app import db from app.activitypub import bp -from flask import request, Response, render_template, current_app, abort, jsonify +from flask import request, Response, render_template, current_app, abort, jsonify, json +from app.activitypub.signature import HttpSignature from app.community.routes import show_community -from app.models import User, Community +from app.models import User, Community, CommunityJoinRequest, CommunityMember, CommunityBan from app.activitypub.util import public_key, users_total, active_half_year, active_month, local_posts, local_comments, \ - post_to_activity + post_to_activity, find_actor_or_create INBOX = [] @@ -155,13 +156,7 @@ def community_profile(actor): server = current_app.config['SERVER_NAME'] actor_data = {"@context": [ "https://www.w3.org/ns/activitystreams", - "https://w3id.org/security/v1", - { - "manuallyApprovesFollowers": "as:manuallyApprovesFollowers", - "schema": "http://schema.org#", - "PropertyValue": "schema:PropertyValue", - "value": "schema:value" - } + "https://w3id.org/security/v1" ], "type": "Group", "id": f"https://{server}/c/{actor}", @@ -202,6 +197,36 @@ def community_profile(actor): abort(404) +@bp.route('/inbox', methods=['GET', 'POST']) +def shared_inbox(): + if request.method == 'POST': + request_json = request.get_json() + actor = find_actor_or_create(request_json['actor']) + if actor is not None: + if HttpSignature.verify_request(request, actor.public_key, skip_date=True): + if 'type' in request_json: + if request_json['type'] == 'Announce': + ... + elif request_json['type'] == 'Follow': + # todo: send accept message if not banned + banned = CommunityBan.query.filter_by(user_id=current_user.id, + community_id=community.id).first() + ... + elif request_json['type'] == 'Accept': + if request_json['object']['type'] == 'Follow': + community_ap_id = request_json['actor'] + user_ap_id = request_json['object']['actor'] + user = find_actor_or_create(user_ap_id) + community = find_actor_or_create(community_ap_id) + join_request = CommunityJoinRequest.query.filter_by(user_id=user.id, + community_id=community.id).first() + if join_request: + member = CommunityMember(user_id=user.id, community_id=community.id) + db.session.add(member) + db.session.commit() + + + @bp.route('/c//outbox', methods=['GET']) def community_outbox(actor): actor = actor.strip() @@ -213,24 +238,6 @@ def community_outbox(actor): "@context": [ "https://www.w3.org/ns/activitystreams", "https://w3id.org/security/v1", - { - "lemmy": "https://join-lemmy.org/ns#", - "pt": "https://joinpeertube.org/ns#", - "sc": "http://schema.org/", - "commentsEnabled": "pt:commentsEnabled", - "sensitive": "as:sensitive", - "postingRestrictedToMods": "lemmy:postingRestrictedToMods", - "removeData": "lemmy:removeData", - "stickied": "lemmy:stickied", - "moderators": { - "@type": "@id", - "@id": "lemmy:moderators" - }, - "expires": "as:endTime", - "distinguished": "lemmy:distinguished", - "language": "sc:inLanguage", - "identifier": "sc:identifier" - } ], "type": "OrderedCollection", "id": f"https://{current_app.config['SERVER_NAME']}/c/{actor}/outbox", diff --git a/app/activitypub/util.py b/app/activitypub/util.py index 771587c2..61b3e238 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -1,9 +1,12 @@ import json import os +from datetime import datetime +from typing import Union +import markdown2 from flask import current_app from sqlalchemy import text from app import db -from app.models import User, Post, Community, BannedInstances +from app.models import User, Post, Community, BannedInstances, File import time import base64 import requests @@ -13,6 +16,8 @@ from app.constants import * import functools from urllib.parse import urlparse +from app.utils import get_request + def public_key(): if not os.path.exists('./public.pem'): @@ -49,7 +54,6 @@ def local_comments(): def send_activity(sender: User, host: str, content: str): - date = time.strftime('%a, %d %b %Y %H:%M:%S UTC', time.gmtime()) private_key = serialization.load_pem_private_key(sender.private_key, password=None) @@ -168,8 +172,9 @@ def validate_header_signature(body: str, host: str, date: str, signature: str) - user = find_actor_or_create(body['actor']) return verify_signature(user.private_key, signature, headers) + def banned_user_agents(): - return [] # todo: finish this function + return [] # todo: finish this function @functools.lru_cache(maxsize=100) @@ -178,13 +183,124 @@ def instance_blocked(host): return instance is not None -def find_actor_or_create(actor): +def find_actor_or_create(actor: str) -> Union[User, Community, None]: + user = None + # actor parameter must be formatted as https://server/u/actor or https://server/c/actor if current_app.config['SERVER_NAME'] + '/c/' in actor: - return Community.query.filter_by(name=actor).first() # finds communities formatted like https://localhost/c/* + return Community.query.filter_by( + ap_profile_id=actor).first() # finds communities formatted like https://localhost/c/* - user = User.query.filter_by(ap_profile_id=actor).first() # finds users formatted like https://kbin.social/u/tables + if current_app.config['SERVER_NAME'] + '/u/' in actor: + user = User.query.filter_by(username=actor.split('/')[-1], ap_id=None).first() # finds local users + if user is None: + return None + elif actor.startswith('https://'): + server, address = extract_domain_and_actor(actor) + if instance_blocked(server): + return None + user = User.query.filter_by(ap_profile_id=actor).first() # finds users formatted like https://kbin.social/u/tables + if user is None: + user = Community.query.filter_by(ap_profile_id=actor).first() if user is None: - # todo: retrieve user details via webfinger, etc - ... + # retrieve user details via webfinger, etc + # todo: try, except block around every get_request + webfinger_data = get_request(f"https://{server}/.well-known/webfinger", + params={'resource': f"acct:{address}@{server}"}) + if webfinger_data.status_code == 200: + webfinger_json = webfinger_data.json() + for links in webfinger_json['links']: + if 'rel' in links and links['rel'] == 'self': # this contains the URL of the activitypub profile + type = links['type'] if 'type' in links else 'application/activity+json' + # retrieve the activitypub profile + actor_data = get_request(links['href'], headers={'Accept': type}) + # to see the structure of the json contained in actor_data, do a GET to https://lemmy.world/c/technology with header Accept: application/activity+json + if actor_data.status_code == 200: + activity_json = actor_data.json() + if activity_json['type'] == 'Person': + user = User(user_name=activity_json['preferredUsername'], + email=f"{address}@{server}", + about=parse_summary(activity_json), + created_at=activity_json['published'], + ap_id=f"{address}@{server}", + ap_public_url=activity_json['id'], + ap_profile_id=activity_json['id'], + ap_inbox_url=activity_json['endpoints']['sharedInbox'], + ap_preferred_username=activity_json['preferredUsername'], + ap_fetched_at=datetime.utcnow(), + ap_domain=server, + public_key=activity_json['publicKey']['publicKeyPem'], + # language=community_json['language'][0]['identifier'] # todo: language + ) + if 'icon' in activity_json: + # todo: retrieve icon, save to disk, save more complete File record + avatar = File(source_url=activity_json['icon']['url']) + user.avatar = avatar + db.session.add(avatar) + if 'image' in activity_json: + # todo: retrieve image, save to disk, save more complete File record + cover = File(source_url=activity_json['image']['url']) + user.cover = cover + db.session.add(cover) + db.session.add(user) + db.session.commit() + return user + elif activity_json['type'] == 'Group': + community = Community(name=activity_json['preferredUsername'], + title=activity_json['name'], + description=activity_json['summary'], + nsfw=activity_json['sensitive'], + restricted_to_mods=activity_json['postingRestrictedToMods'], + created_at=activity_json['published'], + last_active=activity_json['updated'], + ap_id=f"{address[1:]}", + ap_public_url=activity_json['id'], + ap_profile_id=activity_json['id'], + ap_followers_url=activity_json['followers'], + ap_inbox_url=activity_json['endpoints']['sharedInbox'], + ap_fetched_at=datetime.utcnow(), + ap_domain=server, + public_key=activity_json['publicKey']['publicKeyPem'], + # language=community_json['language'][0]['identifier'] # todo: language + ) + if 'icon' in activity_json: + # todo: retrieve icon, save to disk, save more complete File record + icon = File(source_url=activity_json['icon']['url']) + community.icon = icon + db.session.add(icon) + if 'image' in activity_json: + # todo: retrieve image, save to disk, save more complete File record + image = File(source_url=activity_json['image']['url']) + community.image = image + db.session.add(image) + db.session.add(community) + db.session.commit() + return community + return None else: return user + + +def extract_domain_and_actor(url_string: str): + # Parse the URL + parsed_url = urlparse(url_string) + + # Extract the server domain name + server_domain = parsed_url.netloc + + # Extract the part of the string after the last '/' character + actor = parsed_url.path.split('/')[-1] + + return server_domain, actor + + +# create a summary from markdown if present, otherwise use html if available +def parse_summary(user_json) -> str: + if 'source' in user_json and user_json['source'].get('mediaType') == 'text/markdown': + # Convert Markdown to HTML + markdown_text = user_json['source']['content'] + html_content = markdown2.markdown(markdown_text) + return html_content + elif 'summary' in user_json: + return user_json['summary'] + else: + return '' diff --git a/app/community/routes.py b/app/community/routes.py index 4c87f57d..6a13b146 100644 --- a/app/community/routes.py +++ b/app/community/routes.py @@ -3,11 +3,11 @@ from flask import render_template, redirect, url_for, flash, request, make_respo from flask_login import login_user, logout_user, current_user from flask_babel import _ from app import db -from app.activitypub.signature import RsaKeys +from app.activitypub.signature import RsaKeys, HttpSignature from app.community.forms import SearchRemoteCommunity, AddLocalCommunity from app.community.util import search_for_community, community_url_exists from app.constants import SUBSCRIPTION_MEMBER, SUBSCRIPTION_OWNER -from app.models import User, Community, CommunityMember +from app.models import User, Community, CommunityMember, CommunityJoinRequest, CommunityBan from app.community import bp from app.utils import get_setting from sqlalchemy import or_ @@ -25,7 +25,8 @@ def add_local(): form.url.data = form.url.data[3:] private_key, public_key = RsaKeys.generate_keypair() community = Community(title=form.community_name.data, name=form.url.data, description=form.description.data, - rules=form.rules.data, nsfw=form.nsfw.data, private_key=private_key, public_key=public_key, + rules=form.rules.data, nsfw=form.nsfw.data, private_key=private_key, + public_key=public_key, subscriptions_count=1) db.session.add(community) db.session.commit() @@ -53,7 +54,8 @@ def add_remote(): elif '@' in address: new_community = search_for_community('!' + address) else: - message = Markup('Type address in the format !community@server.name. Search on Lemmyverse.net to find some.') + message = Markup( + 'Type address in the format !community@server.name. Search on Lemmyverse.net to find some.') flash(message, 'error') return render_template('community/add_remote.html', @@ -68,7 +70,7 @@ def show_community(community: Community): CommunityMember.is_owner, CommunityMember.is_moderator )) - ).all() + ).all() is_moderator = any(mod.user_id == current_user.id for mod in mods) is_owner = any(mod.user_id == current_user.id and mod.is_owner == True for mod in mods) @@ -85,18 +87,48 @@ def show_community(community: Community): @bp.route('//subscribe', methods=['GET']) def subscribe(actor): + remote = False actor = actor.strip() if '@' in actor: community = Community.query.filter_by(banned=False, ap_id=actor).first() + remote = True else: community = Community.query.filter_by(name=actor, banned=False, ap_id=None).first() if community is not None: if not current_user.subscribed(community): - membership = CommunityMember(user_id=current_user.id, community_id=community.id) - db.session.add(membership) - db.session.commit() - flash('You have subscribed to ' + community.title) + if remote: + # send ActivityPub message to remote community, asking to follow. Accept message will be sent to our shared inbox + join_request = CommunityJoinRequest(user_id=current_user.id, community_id=community.id) + db.session.add(join_request) + db.session.commit() + follow = { + "actor": f"https://{current_app.config['SERVER_NAME']}/u/{current_user.user_name}", + "to": [community.ap_id], + "object": community.ap_id, + "type": "Follow", + "id": f"https://{current_app.config['SERVER_NAME']}/activities/follow/" + join_request.id + } + try: + message = HttpSignature.signed_request(community.ap_inbox_url, follow, current_user.private_key, + current_user.ap_profile_id + '#main-key') + if message.status_code == 200: + flash('Your request to subscribe has been sent to ' + community.title) + else: + flash('Response status code was not 200', 'warning') + current_app.logger.error('Response code for subscription attempt was ' + + str(message.status_code) + ' ' + message.text) + except Exception as ex: + flash('Failed to send request to subscribe: ' + str(ex), 'error') + current_app.logger.error("Exception while trying to subscribe" + str(ex)) + else: # for local communities, joining is instant + banned = CommunityBan.query.filter_by(user_id=current_user.id, community_id=community.id).first() + if banned: + flash('You cannot join this community') + member = CommunityMember(user_id=current_user.id, community_id=community.id) + db.session.add(member) + db.session.commit() + flash('You are subscribed to ' + community.title) referrer = request.headers.get('Referer', None) if referrer is not None: return redirect(referrer) diff --git a/app/constants.py b/app/constants.py index 1bdcbc49..d4a274f6 100644 --- a/app/constants.py +++ b/app/constants.py @@ -11,3 +11,5 @@ SUBSCRIPTION_OWNER = 3 SUBSCRIPTION_MODERATOR = 2 SUBSCRIPTION_MEMBER = 1 SUBSCRIPTION_NONMEMBER = 0 +SUBSCRIPTION_BANNED = -1 + diff --git a/app/models.py b/app/models.py index 53464e10..3899ca25 100644 --- a/app/models.py +++ b/app/models.py @@ -10,7 +10,8 @@ from sqlalchemy_utils.types import TSVectorType # https://sqlalchemy-searchable. from app import db, login import jwt -from app.constants import SUBSCRIPTION_NONMEMBER, SUBSCRIPTION_MEMBER, SUBSCRIPTION_MODERATOR, SUBSCRIPTION_OWNER +from app.constants import SUBSCRIPTION_NONMEMBER, SUBSCRIPTION_MEMBER, SUBSCRIPTION_MODERATOR, SUBSCRIPTION_OWNER, \ + SUBSCRIPTION_BANNED class File(db.Model): @@ -121,9 +122,12 @@ class User(UserMixin, db.Model): searchable = db.Column(db.Boolean, default=True) indexable = db.Column(db.Boolean, default=False) - ap_id = db.Column(db.String(255), index=True) - ap_profile_id = db.Column(db.String(255)) - ap_public_url = db.Column(db.String(255)) + avatar = db.relationship('File', foreign_keys=[avatar_id], single_parent=True, cascade="all, delete-orphan") + cover = db.relationship('File', foreign_keys=[cover_id], single_parent=True, cascade="all, delete-orphan") + + ap_id = db.Column(db.String(255), index=True) # e.g. username@server + ap_profile_id = db.Column(db.String(255), index=True) # e.g. https://server/u/username + ap_public_url = db.Column(db.String(255)) # e.g. https://server/u/username ap_fetched_at = db.Column(db.DateTime) ap_followers_url = db.Column(db.String(255)) ap_preferred_username = db.Column(db.String(255)) @@ -134,7 +138,6 @@ class User(UserMixin, db.Model): search_vector = db.Column(TSVectorType('user_name', 'bio', 'keywords')) activity = db.relationship('ActivityLog', backref='account', lazy='dynamic', cascade="all, delete-orphan") - avatar = db.relationship(File, foreign_keys=[avatar_id], cascade="all, delete-orphan") posts = db.relationship('Post', backref='author', lazy='dynamic', cascade="all, delete-orphan") post_replies = db.relationship('PostReply', backref='author', lazy='dynamic', cascade="all, delete-orphan") @@ -186,7 +189,9 @@ class User(UserMixin, db.Model): return False subscription:CommunityMember = CommunityMember.query.filter_by(user_id=self.id, community_id=community.id).first() if subscription: - if subscription.is_owner: + if subscription.is_banned: + return SUBSCRIPTION_BANNED + elif subscription.is_owner: return SUBSCRIPTION_OWNER elif subscription.is_moderator: return SUBSCRIPTION_MODERATOR @@ -305,6 +310,15 @@ class CommunityMember(db.Model): created_at = db.Column(db.DateTime, default=datetime.utcnow) +class CommunityBan(db.Model): + user_id = db.Column(db.Integer, db.ForeignKey('user.id'), primary_key=True) + community_id = db.Column(db.Integer, db.ForeignKey('community.id'), primary_key=True) + banned_by = db.Column(db.Integer, db.ForeignKey('user.id')) + reason = db.Column(db.String(50)) + created_at = db.Column(db.DateTime, default=datetime.utcnow) + ban_until = db.Column(db.DateTime) + + class UserNote(db.Model): id = db.Column(db.Integer, primary_key=True) user_id = db.Column(db.Integer, db.ForeignKey('user.id')) @@ -347,6 +361,18 @@ class Interest(db.Model): communities = db.Column(db.Text) +class CommunityJoinRequest(db.Model): + id = db.Column(db.Integer, primary_key=True) + user_id = db.Column(db.Integer, db.ForeignKey('user.id')) + community_id = db.Column(db.Integer, db.ForeignKey('community.id')) + + +class UserFollowRequest(db.Model): + id = db.Column(db.Integer, primary_key=True) + user_id = db.Column(db.Integer, db.ForeignKey('user.id')) + follow_id = db.Column(db.Integer, db.ForeignKey('user.id')) + + @login.user_loader def load_user(id): return User.query.get(int(id)) diff --git a/app/templates/community/community.html b/app/templates/community/community.html index 982740b9..36186036 100644 --- a/app/templates/community/community.html +++ b/app/templates/community/community.html @@ -48,7 +48,7 @@

{{ community.description }}

{{ community.rules }}

- {% if len(mods) > 0 %} + {% if len(mods) > 0 and not community.private_mods %}

Moderators

    {% for mod in mods %} diff --git a/migrations/versions/251be00ae302_follow_requests.py b/migrations/versions/251be00ae302_follow_requests.py new file mode 100644 index 00000000..96910d09 --- /dev/null +++ b/migrations/versions/251be00ae302_follow_requests.py @@ -0,0 +1,50 @@ +"""follow requests + +Revision ID: 251be00ae302 +Revises: 01107dfe5a29 +Create Date: 2023-09-08 19:07:51.474128 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '251be00ae302' +down_revision = '01107dfe5a29' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('community_join_request', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=True), + sa.Column('community_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['community_id'], ['community.id'], ), + sa.ForeignKeyConstraint(['user_id'], ['user.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('user_follow_request', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=True), + sa.Column('follow_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['follow_id'], ['user.id'], ), + sa.ForeignKeyConstraint(['user_id'], ['user.id'], ), + sa.PrimaryKeyConstraint('id') + ) + with op.batch_alter_table('user', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_user_ap_profile_id'), ['ap_profile_id'], unique=False) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('user', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_user_ap_profile_id')) + + op.drop_table('user_follow_request') + op.drop_table('community_join_request') + # ### end Alembic commands ### diff --git a/migrations/versions/cc98a471a1ad_community_ban.py b/migrations/versions/cc98a471a1ad_community_ban.py new file mode 100644 index 00000000..e03e901e --- /dev/null +++ b/migrations/versions/cc98a471a1ad_community_ban.py @@ -0,0 +1,39 @@ +"""community ban + +Revision ID: cc98a471a1ad +Revises: 251be00ae302 +Create Date: 2023-09-08 20:03:14.527356 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'cc98a471a1ad' +down_revision = '251be00ae302' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('community_ban', + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('community_id', sa.Integer(), nullable=False), + sa.Column('banned_by', sa.Integer(), nullable=True), + sa.Column('reason', sa.String(length=50), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('ban_until', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['banned_by'], ['user.id'], ), + sa.ForeignKeyConstraint(['community_id'], ['community.id'], ), + sa.ForeignKeyConstraint(['user_id'], ['user.id'], ), + sa.PrimaryKeyConstraint('user_id', 'community_id') + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('community_ban') + # ### end Alembic commands ### diff --git a/requirements.txt b/requirements.txt index 3ede490e..a2b926e6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,3 +19,4 @@ pycryptodome==3.18.0 arrow==1.2.3 pyld==2.0.3 boto3==1.28.35 +markdown2==2.4.10