follow remote communities - activitypub

CONTRIBUTING.md

@@ -1,6 +1,6 @@
 # Contributing to PyFedi
 
 Please discuss your ideas in an issue at https://codeberg.org/rimu/pyfedi/issues before 
-starting any work to ensure alignment with the roadmap, architecture and processes.
+starting any large pieces of work to ensure alignment with the roadmap, architecture and processes.
 
 Mailing list, Matrix channel, etc still to come.

README.md

@@ -6,11 +6,3 @@ A lemmy/kbin clone written in Python with Flask.
  - Easy setup, easy to manage - few dependencies and extra software required.
  - GPL.
 
-## Contributing
-
-Join our Maxtrix channel.
-
-Report bugs in our issue queue.
-
-If submitting a substantial PR, communicate your intentions to the maintainer(s) before investing your effort into coding. 
-This is to ensure your contribution will fit well with the rest of the code base. 

app/activitypub/routes.py

@@ -2,12 +2,13 @@ from sqlalchemy import text
 
 from app import db
 from app.activitypub import bp
-from flask import request, Response, render_template, current_app, abort, jsonify
+from flask import request, Response, render_template, current_app, abort, jsonify, json
 
+from app.activitypub.signature import HttpSignature
 from app.community.routes import show_community
-from app.models import User, Community
+from app.models import User, Community, CommunityJoinRequest, CommunityMember, CommunityBan
 from app.activitypub.util import public_key, users_total, active_half_year, active_month, local_posts, local_comments, \
-    post_to_activity
+    post_to_activity, find_actor_or_create
 
 INBOX = []
 
@@ -155,13 +156,7 @@ def community_profile(actor):
             server = current_app.config['SERVER_NAME']
             actor_data = {"@context": [
                 "https://www.w3.org/ns/activitystreams",
-                "https://w3id.org/security/v1",
+                "https://w3id.org/security/v1"
-                {
-                    "manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
-                    "schema": "http://schema.org#",
-                    "PropertyValue": "schema:PropertyValue",
-                    "value": "schema:value"
-                }
                 ],
                 "type": "Group",
                 "id": f"https://{server}/c/{actor}",
@@ -202,6 +197,36 @@ def community_profile(actor):
         abort(404)
 
 
+@bp.route('/inbox', methods=['GET', 'POST'])
+def shared_inbox():
+    if request.method == 'POST':
+        request_json = request.get_json()
+        actor = find_actor_or_create(request_json['actor'])
+        if actor is not None:
+            if HttpSignature.verify_request(request, actor.public_key, skip_date=True):
+                if 'type' in request_json:
+                    if request_json['type'] == 'Announce':
+                        ...
+                    elif request_json['type'] == 'Follow':
+                        # todo: send accept message if not banned
+                        banned = CommunityBan.query.filter_by(user_id=current_user.id,
+                                                              community_id=community.id).first()
+                        ...
+                    elif request_json['type'] == 'Accept':
+                        if request_json['object']['type'] == 'Follow':
+                            community_ap_id = request_json['actor']
+                            user_ap_id = request_json['object']['actor']
+                            user = find_actor_or_create(user_ap_id)
+                            community = find_actor_or_create(community_ap_id)
+                            join_request = CommunityJoinRequest.query.filter_by(user_id=user.id,
+                                                                                community_id=community.id).first()
+                            if join_request:
+                                member = CommunityMember(user_id=user.id, community_id=community.id)
+                                db.session.add(member)
+                                db.session.commit()
+
+
+
 @bp.route('/c/<actor>/outbox', methods=['GET'])
 def community_outbox(actor):
     actor = actor.strip()
@@ -213,24 +238,6 @@ def community_outbox(actor):
             "@context": [
                 "https://www.w3.org/ns/activitystreams",
                 "https://w3id.org/security/v1",
-                {
-                    "lemmy": "https://join-lemmy.org/ns#",
-                    "pt": "https://joinpeertube.org/ns#",
-                    "sc": "http://schema.org/",
-                    "commentsEnabled": "pt:commentsEnabled",
-                    "sensitive": "as:sensitive",
-                    "postingRestrictedToMods": "lemmy:postingRestrictedToMods",
-                    "removeData": "lemmy:removeData",
-                    "stickied": "lemmy:stickied",
-                    "moderators": {
-                        "@type": "@id",
-                        "@id": "lemmy:moderators"
-                    },
-                    "expires": "as:endTime",
-                    "distinguished": "lemmy:distinguished",
-                    "language": "sc:inLanguage",
-                    "identifier": "sc:identifier"
-                }
             ],
             "type": "OrderedCollection",
             "id": f"https://{current_app.config['SERVER_NAME']}/c/{actor}/outbox",

app/activitypub/util.py

@@ -1,9 +1,12 @@
 import json
 import os
+from datetime import datetime
+from typing import Union
+import markdown2
 from flask import current_app
 from sqlalchemy import text
 from app import db
-from app.models import User, Post, Community, BannedInstances
+from app.models import User, Post, Community, BannedInstances, File
 import time
 import base64
 import requests
@@ -13,6 +16,8 @@ from app.constants import *
 import functools
 from urllib.parse import urlparse
 
+from app.utils import get_request
+
 
 def public_key():
     if not os.path.exists('./public.pem'):
@@ -49,7 +54,6 @@ def local_comments():
 
 
 def send_activity(sender: User, host: str, content: str):
-
     date = time.strftime('%a, %d %b %Y %H:%M:%S UTC', time.gmtime())
 
     private_key = serialization.load_pem_private_key(sender.private_key, password=None)
@@ -168,8 +172,9 @@ def validate_header_signature(body: str, host: str, date: str, signature: str) -
     user = find_actor_or_create(body['actor'])
     return verify_signature(user.private_key, signature, headers)
 
+
 def banned_user_agents():
-    return []   # todo: finish this function
+    return []  # todo: finish this function
 
 
 @functools.lru_cache(maxsize=100)
@@ -178,13 +183,124 @@ def instance_blocked(host):
     return instance is not None
 
 
-def find_actor_or_create(actor):
+def find_actor_or_create(actor: str) -> Union[User, Community, None]:
+    user = None
+    # actor parameter must be formatted as https://server/u/actor or https://server/c/actor
     if current_app.config['SERVER_NAME'] + '/c/' in actor:
-        return Community.query.filter_by(name=actor).first()  # finds communities formatted like https://localhost/c/*
+        return Community.query.filter_by(
+            ap_profile_id=actor).first()  # finds communities formatted like https://localhost/c/*
 
-    user = User.query.filter_by(ap_profile_id=actor).first() # finds users formatted like https://kbin.social/u/tables
+    if current_app.config['SERVER_NAME'] + '/u/' in actor:
+        user = User.query.filter_by(username=actor.split('/')[-1], ap_id=None).first()  # finds local users
+        if user is None:
+            return None
+    elif actor.startswith('https://'):
+        server, address = extract_domain_and_actor(actor)
+        if instance_blocked(server):
+            return None
+        user = User.query.filter_by(ap_profile_id=actor).first()  # finds users formatted like https://kbin.social/u/tables
+        if user is None:
+            user = Community.query.filter_by(ap_profile_id=actor).first()
     if user is None:
-        # todo: retrieve user details via webfinger, etc
+        # retrieve user details via webfinger, etc
-        ...
+        # todo: try, except block around every get_request
+        webfinger_data = get_request(f"https://{server}/.well-known/webfinger",
+                                     params={'resource': f"acct:{address}@{server}"})
+        if webfinger_data.status_code == 200:
+            webfinger_json = webfinger_data.json()
+            for links in webfinger_json['links']:
+                if 'rel' in links and links['rel'] == 'self':  # this contains the URL of the activitypub profile
+                    type = links['type'] if 'type' in links else 'application/activity+json'
+                    # retrieve the activitypub profile
+                    actor_data = get_request(links['href'], headers={'Accept': type})
+                    # to see the structure of the json contained in actor_data, do a GET to https://lemmy.world/c/technology with header Accept: application/activity+json
+                    if actor_data.status_code == 200:
+                        activity_json = actor_data.json()
+                        if activity_json['type'] == 'Person':
+                            user = User(user_name=activity_json['preferredUsername'],
+                                        email=f"{address}@{server}",
+                                        about=parse_summary(activity_json),
+                                        created_at=activity_json['published'],
+                                        ap_id=f"{address}@{server}",
+                                        ap_public_url=activity_json['id'],
+                                        ap_profile_id=activity_json['id'],
+                                        ap_inbox_url=activity_json['endpoints']['sharedInbox'],
+                                        ap_preferred_username=activity_json['preferredUsername'],
+                                        ap_fetched_at=datetime.utcnow(),
+                                        ap_domain=server,
+                                        public_key=activity_json['publicKey']['publicKeyPem'],
+                                        # language=community_json['language'][0]['identifier'] # todo: language
+                                        )
+                            if 'icon' in activity_json:
+                                # todo: retrieve icon, save to disk, save more complete File record
+                                avatar = File(source_url=activity_json['icon']['url'])
+                                user.avatar = avatar
+                                db.session.add(avatar)
+                            if 'image' in activity_json:
+                                # todo: retrieve image, save to disk, save more complete File record
+                                cover = File(source_url=activity_json['image']['url'])
+                                user.cover = cover
+                                db.session.add(cover)
+                            db.session.add(user)
+                            db.session.commit()
+                            return user
+                        elif activity_json['type'] == 'Group':
+                            community = Community(name=activity_json['preferredUsername'],
+                                                  title=activity_json['name'],
+                                                  description=activity_json['summary'],
+                                                  nsfw=activity_json['sensitive'],
+                                                  restricted_to_mods=activity_json['postingRestrictedToMods'],
+                                                  created_at=activity_json['published'],
+                                                  last_active=activity_json['updated'],
+                                                  ap_id=f"{address[1:]}",
+                                                  ap_public_url=activity_json['id'],
+                                                  ap_profile_id=activity_json['id'],
+                                                  ap_followers_url=activity_json['followers'],
+                                                  ap_inbox_url=activity_json['endpoints']['sharedInbox'],
+                                                  ap_fetched_at=datetime.utcnow(),
+                                                  ap_domain=server,
+                                                  public_key=activity_json['publicKey']['publicKeyPem'],
+                                                  # language=community_json['language'][0]['identifier'] # todo: language
+                                                  )
+                            if 'icon' in activity_json:
+                                # todo: retrieve icon, save to disk, save more complete File record
+                                icon = File(source_url=activity_json['icon']['url'])
+                                community.icon = icon
+                                db.session.add(icon)
+                            if 'image' in activity_json:
+                                # todo: retrieve image, save to disk, save more complete File record
+                                image = File(source_url=activity_json['image']['url'])
+                                community.image = image
+                                db.session.add(image)
+                            db.session.add(community)
+                            db.session.commit()
+                            return community
+        return None
     else:
         return user
+
+
+def extract_domain_and_actor(url_string: str):
+    # Parse the URL
+    parsed_url = urlparse(url_string)
+
+    # Extract the server domain name
+    server_domain = parsed_url.netloc
+
+    # Extract the part of the string after the last '/' character
+    actor = parsed_url.path.split('/')[-1]
+
+    return server_domain, actor
+
+
+# create a summary from markdown if present, otherwise use html if available
+def parse_summary(user_json) -> str:
+    if 'source' in user_json and user_json['source'].get('mediaType') == 'text/markdown':
+        # Convert Markdown to HTML
+        markdown_text = user_json['source']['content']
+        html_content = markdown2.markdown(markdown_text)
+        return html_content
+    elif 'summary' in user_json:
+        return user_json['summary']
+    else:
+        return ''

app/community/routes.py

@@ -3,11 +3,11 @@ from flask import render_template, redirect, url_for, flash, request, make_respo
 from flask_login import login_user, logout_user, current_user
 from flask_babel import _
 from app import db
-from app.activitypub.signature import RsaKeys
+from app.activitypub.signature import RsaKeys, HttpSignature
 from app.community.forms import SearchRemoteCommunity, AddLocalCommunity
 from app.community.util import search_for_community, community_url_exists
 from app.constants import SUBSCRIPTION_MEMBER, SUBSCRIPTION_OWNER
-from app.models import User, Community, CommunityMember
+from app.models import User, Community, CommunityMember, CommunityJoinRequest, CommunityBan
 from app.community import bp
 from app.utils import get_setting
 from sqlalchemy import or_
@@ -25,7 +25,8 @@ def add_local():
             form.url.data = form.url.data[3:]
         private_key, public_key = RsaKeys.generate_keypair()
         community = Community(title=form.community_name.data, name=form.url.data, description=form.description.data,
-                              rules=form.rules.data, nsfw=form.nsfw.data, private_key=private_key, public_key=public_key,
+                              rules=form.rules.data, nsfw=form.nsfw.data, private_key=private_key,
+                              public_key=public_key,
                               subscriptions_count=1)
         db.session.add(community)
         db.session.commit()
@@ -53,7 +54,8 @@ def add_remote():
         elif '@' in address:
             new_community = search_for_community('!' + address)
         else:
-            message = Markup('Type address in the format !community@server.name. Search on <a href="https://lemmyverse.net/communities">Lemmyverse.net</a> to find some.')
+            message = Markup(
+                'Type address in the format !community@server.name. Search on <a href="https://lemmyverse.net/communities">Lemmyverse.net</a> to find some.')
             flash(message, 'error')
 
     return render_template('community/add_remote.html',
@@ -68,7 +70,7 @@ def show_community(community: Community):
                                             CommunityMember.is_owner,
                                             CommunityMember.is_moderator
                                         ))
-    ).all()
+                                        ).all()
 
     is_moderator = any(mod.user_id == current_user.id for mod in mods)
     is_owner = any(mod.user_id == current_user.id and mod.is_owner == True for mod in mods)
@@ -85,18 +87,48 @@ def show_community(community: Community):
 
 @bp.route('/<actor>/subscribe', methods=['GET'])
 def subscribe(actor):
+    remote = False
     actor = actor.strip()
     if '@' in actor:
         community = Community.query.filter_by(banned=False, ap_id=actor).first()
+        remote = True
     else:
         community = Community.query.filter_by(name=actor, banned=False, ap_id=None).first()
 
     if community is not None:
         if not current_user.subscribed(community):
-            membership = CommunityMember(user_id=current_user.id, community_id=community.id)
+            if remote:
-            db.session.add(membership)
+                # send ActivityPub message to remote community, asking to follow. Accept message will be sent to our shared inbox
-            db.session.commit()
+                join_request = CommunityJoinRequest(user_id=current_user.id, community_id=community.id)
-        flash('You have subscribed to ' + community.title)
+                db.session.add(join_request)
+                db.session.commit()
+                follow = {
+                    "actor": f"https://{current_app.config['SERVER_NAME']}/u/{current_user.user_name}",
+                    "to": [community.ap_id],
+                    "object": community.ap_id,
+                    "type": "Follow",
+                    "id": f"https://{current_app.config['SERVER_NAME']}/activities/follow/" + join_request.id
+                }
+                try:
+                    message = HttpSignature.signed_request(community.ap_inbox_url, follow, current_user.private_key,
+                                                           current_user.ap_profile_id + '#main-key')
+                    if message.status_code == 200:
+                        flash('Your request to subscribe has been sent to ' + community.title)
+                    else:
+                        flash('Response status code was not 200', 'warning')
+                        current_app.logger.error('Response code for subscription attempt was ' +
+                                                 str(message.status_code) + ' ' + message.text)
+                except Exception as ex:
+                    flash('Failed to send request to subscribe: ' + str(ex), 'error')
+                    current_app.logger.error("Exception while trying to subscribe" + str(ex))
+            else:   # for local communities, joining is instant
+                banned = CommunityBan.query.filter_by(user_id=current_user.id, community_id=community.id).first()
+                if banned:
+                    flash('You cannot join this community')
+                member = CommunityMember(user_id=current_user.id, community_id=community.id)
+                db.session.add(member)
+                db.session.commit()
+                flash('You are subscribed to ' + community.title)
         referrer = request.headers.get('Referer', None)
         if referrer is not None:
             return redirect(referrer)

app/constants.py

@@ -11,3 +11,5 @@ SUBSCRIPTION_OWNER = 3
 SUBSCRIPTION_MODERATOR = 2
 SUBSCRIPTION_MEMBER = 1
 SUBSCRIPTION_NONMEMBER = 0
+SUBSCRIPTION_BANNED = -1
+

app/models.py

@@ -10,7 +10,8 @@ from sqlalchemy_utils.types import TSVectorType # https://sqlalchemy-searchable.
 from app import db, login
 import jwt
 
-from app.constants import SUBSCRIPTION_NONMEMBER, SUBSCRIPTION_MEMBER, SUBSCRIPTION_MODERATOR, SUBSCRIPTION_OWNER
+from app.constants import SUBSCRIPTION_NONMEMBER, SUBSCRIPTION_MEMBER, SUBSCRIPTION_MODERATOR, SUBSCRIPTION_OWNER, \
+    SUBSCRIPTION_BANNED
 
 
 class File(db.Model):
@@ -121,9 +122,12 @@ class User(UserMixin, db.Model):
     searchable = db.Column(db.Boolean, default=True)
     indexable = db.Column(db.Boolean, default=False)
 
-    ap_id = db.Column(db.String(255), index=True)
+    avatar = db.relationship('File', foreign_keys=[avatar_id], single_parent=True, cascade="all, delete-orphan")
-    ap_profile_id = db.Column(db.String(255))
+    cover = db.relationship('File', foreign_keys=[cover_id], single_parent=True, cascade="all, delete-orphan")
-    ap_public_url = db.Column(db.String(255))
+
+    ap_id = db.Column(db.String(255), index=True)           # e.g. username@server
+    ap_profile_id = db.Column(db.String(255), index=True)   # e.g. https://server/u/username
+    ap_public_url = db.Column(db.String(255))               # e.g. https://server/u/username
     ap_fetched_at = db.Column(db.DateTime)
     ap_followers_url = db.Column(db.String(255))
     ap_preferred_username = db.Column(db.String(255))
@@ -134,7 +138,6 @@ class User(UserMixin, db.Model):
 
     search_vector = db.Column(TSVectorType('user_name', 'bio', 'keywords'))
     activity = db.relationship('ActivityLog', backref='account', lazy='dynamic', cascade="all, delete-orphan")
-    avatar = db.relationship(File, foreign_keys=[avatar_id], cascade="all, delete-orphan")
     posts = db.relationship('Post', backref='author', lazy='dynamic', cascade="all, delete-orphan")
     post_replies = db.relationship('PostReply', backref='author', lazy='dynamic', cascade="all, delete-orphan")
 
@@ -186,7 +189,9 @@ class User(UserMixin, db.Model):
             return False
         subscription:CommunityMember = CommunityMember.query.filter_by(user_id=self.id, community_id=community.id).first()
         if subscription:
-            if subscription.is_owner:
+            if subscription.is_banned:
+                return SUBSCRIPTION_BANNED
+            elif subscription.is_owner:
                 return SUBSCRIPTION_OWNER
             elif subscription.is_moderator:
                 return SUBSCRIPTION_MODERATOR
@@ -305,6 +310,15 @@ class CommunityMember(db.Model):
     created_at = db.Column(db.DateTime, default=datetime.utcnow)
 
 
+class CommunityBan(db.Model):
+    user_id = db.Column(db.Integer, db.ForeignKey('user.id'), primary_key=True)
+    community_id = db.Column(db.Integer, db.ForeignKey('community.id'), primary_key=True)
+    banned_by = db.Column(db.Integer, db.ForeignKey('user.id'))
+    reason = db.Column(db.String(50))
+    created_at = db.Column(db.DateTime, default=datetime.utcnow)
+    ban_until = db.Column(db.DateTime)
+
+
 class UserNote(db.Model):
     id = db.Column(db.Integer, primary_key=True)
     user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
@@ -347,6 +361,18 @@ class Interest(db.Model):
     communities = db.Column(db.Text)
 
 
+class CommunityJoinRequest(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
+    community_id = db.Column(db.Integer, db.ForeignKey('community.id'))
+
+
+class UserFollowRequest(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
+    follow_id = db.Column(db.Integer, db.ForeignKey('user.id'))
+
+
 @login.user_loader
 def load_user(id):
     return User.query.get(int(id))

app/templates/community/community.html

@@ -48,7 +48,7 @@
             <div class="card-body">
                 <p>{{ community.description }}</p>
                 <p>{{ community.rules }}</p>
-                {% if len(mods) > 0 %}
+                {% if len(mods) > 0 and not community.private_mods %}
                     <h3>Moderators</h3>
                     <ol>
                         {% for mod in mods %}

migrations/versions/251be00ae302_follow_requests.py

@@ -0,0 +1,50 @@
+"""follow requests
+
+Revision ID: 251be00ae302
+Revises: 01107dfe5a29
+Create Date: 2023-09-08 19:07:51.474128
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '251be00ae302'
+down_revision = '01107dfe5a29'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('community_join_request',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('user_id', sa.Integer(), nullable=True),
+    sa.Column('community_id', sa.Integer(), nullable=True),
+    sa.ForeignKeyConstraint(['community_id'], ['community.id'], ),
+    sa.ForeignKeyConstraint(['user_id'], ['user.id'], ),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('user_follow_request',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('user_id', sa.Integer(), nullable=True),
+    sa.Column('follow_id', sa.Integer(), nullable=True),
+    sa.ForeignKeyConstraint(['follow_id'], ['user.id'], ),
+    sa.ForeignKeyConstraint(['user_id'], ['user.id'], ),
+    sa.PrimaryKeyConstraint('id')
+    )
+    with op.batch_alter_table('user', schema=None) as batch_op:
+        batch_op.create_index(batch_op.f('ix_user_ap_profile_id'), ['ap_profile_id'], unique=False)
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('user', schema=None) as batch_op:
+        batch_op.drop_index(batch_op.f('ix_user_ap_profile_id'))
+
+    op.drop_table('user_follow_request')
+    op.drop_table('community_join_request')
+    # ### end Alembic commands ###

migrations/versions/cc98a471a1ad_community_ban.py

@@ -0,0 +1,39 @@
+"""community ban
+
+Revision ID: cc98a471a1ad
+Revises: 251be00ae302
+Create Date: 2023-09-08 20:03:14.527356
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'cc98a471a1ad'
+down_revision = '251be00ae302'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('community_ban',
+    sa.Column('user_id', sa.Integer(), nullable=False),
+    sa.Column('community_id', sa.Integer(), nullable=False),
+    sa.Column('banned_by', sa.Integer(), nullable=True),
+    sa.Column('reason', sa.String(length=50), nullable=True),
+    sa.Column('created_at', sa.DateTime(), nullable=True),
+    sa.Column('ban_until', sa.DateTime(), nullable=True),
+    sa.ForeignKeyConstraint(['banned_by'], ['user.id'], ),
+    sa.ForeignKeyConstraint(['community_id'], ['community.id'], ),
+    sa.ForeignKeyConstraint(['user_id'], ['user.id'], ),
+    sa.PrimaryKeyConstraint('user_id', 'community_id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('community_ban')
+    # ### end Alembic commands ###

requirements.txt

@@ -19,3 +19,4 @@ pycryptodome==3.18.0
 arrow==1.2.3
 pyld==2.0.3
 boto3==1.28.35
+markdown2==2.4.10