federation - post content and replies/comments to remote communities

app/activitypub/routes.py

@@ -6,6 +6,7 @@ from flask import request, Response, current_app, abort, jsonify, json
 
 from app.activitypub.signature import HttpSignature
 from app.community.routes import show_community
+from app.post.routes import continue_discussion, show_post
 from app.user.routes import show_profile
 from app.constants import POST_TYPE_LINK, POST_TYPE_IMAGE, SUBSCRIPTION_MEMBER
 from app.models import User, Community, CommunityJoinRequest, CommunityMember, CommunityBan, ActivityPubLog, Post, \
@@ -132,6 +133,10 @@ def lemmy_federated_instances():
     })
 
 
+def is_activitypub_request():
+    return 'application/ld+json' in request.headers.get('Accept', '') or 'application/activity+json' in request.headers.get('Accept', '')
+
+
 @bp.route('/u/<actor>', methods=['GET'])
 def user_profile(actor):
     """ Requests to this endpoint can be for a JSON representation of the user, or a HTML rendering of their profile.
@@ -143,7 +148,7 @@ def user_profile(actor):
         user = User.query.filter_by(user_name=actor, deleted=False, banned=False, ap_id=None).first()
 
     if user is not None:
-        if 'application/ld+json' in request.headers.get('Accept', '') or 'application/activity+json' in request.headers.get('Accept', ''):
+        if is_activitypub_request():
             server = current_app.config['SERVER_NAME']
             actor_data = {  "@context": default_context(),
                             "type": "Person",
@@ -199,7 +204,7 @@ def community_profile(actor):
     else:
         community: Community = Community.query.filter_by(name=actor, banned=False, ap_id=None).first()
     if community is not None:
-        if 'application/ld+json' in request.headers.get('Accept', '') or 'application/activity+json' in request.headers.get('Accept', ''):
+        if is_activitypub_request():
             server = current_app.config['SERVER_NAME']
             actor_data = {"@context": default_context(),
                 "type": "Group",
@@ -375,7 +380,7 @@ def shared_inbox():
                                             db.session.add(post_reply)
                                             post.reply_count += 1
                                             community.post_reply_count += 1
-                                            community.last_active = datetime.utcnow()
+                                            community.last_active = post.last_active = datetime.utcnow()
                                             activity_log.result = 'success'
                                             db.session.commit()
                                             vote = PostReplyVote(user_id=user.id, author_id=post_reply.user_id, post_reply_id=post_reply.id,
@@ -918,3 +923,56 @@ def inbox(actor):
     if request.method == 'POST':
         INBOX.append(request.data)
         return Response(status=200)
+
+
+@bp.route('/comment/<int:comment_id>', methods=['GET'])
+def comment_ap(comment_id):
+    if is_activitypub_request():
+        reply = PostReply.query.get_or_404(comment_id)
+        reply_data = {
+            "@context": default_context(),
+            "type": "Note",
+            "id": reply.ap_id,
+            "attributedTo": reply.author.profile_id(),
+            "inReplyTo": reply.in_reply_to(),
+            "to": [
+                "https://www.w3.org/ns/activitystreams#Public",
+                reply.to()
+            ],
+            "cc": [
+                reply.community.profile_id(),
+                reply.author.followers_url()
+            ],
+            'content': reply.body_html,
+            'mediaType': 'text/html',
+            'published': ap_datetime(reply.created_at),
+            'distinguished': False,
+            'audience': reply.community.profile_id()
+        }
+        if reply.edited_at:
+            reply_data['updated'] = ap_datetime(reply.edited_at)
+        if reply.body.strip():
+            reply_data['source'] = {
+                'content': reply.body,
+                'mediaType': 'text/markdown'
+            }
+        resp = jsonify(reply_data)
+        resp.content_type = 'application/activity+json'
+        return resp
+    else:
+        reply = PostReply.query.get(comment_id)
+        continue_discussion(reply.post.id, comment_id)
+
+
+@bp.route('/post/<int:post_id>', methods=['GET', 'POST'])
+def post_ap(post_id):
+    if request.method == 'GET' and is_activitypub_request():
+        post = Post.query.get_or_404(post_id)
+        post_data = post_to_activity(post, post.community)
+        post_data = post_data['object']['object']
+        post_data['@context'] = default_context()
+        resp = jsonify(post_data)
+        resp.content_type = 'application/activity+json'
+        return resp
+    else:
+        return show_post(post_id)

app/activitypub/util.py

@@ -14,7 +14,7 @@ from cryptography.hazmat.primitives.asymmetric import padding
 from app.constants import *
 from urllib.parse import urlparse
 
-from app.utils import get_request, allowlist_html, html_to_markdown, get_setting
+from app.utils import get_request, allowlist_html, html_to_markdown, get_setting, ap_datetime
 
 
 def public_key():
@@ -115,7 +115,7 @@ def post_to_activity(post: Post, community: Community):
                 "attachment": [],
                 "commentsEnabled": True,
                 "sensitive": post.nsfw or post.nsfl,
-                "published": post.created_at,
+                "published": ap_datetime(post.created_at),
                 "audience": f"https://{current_app.config['SERVER_NAME']}/c/{community.name}"
             },
             "cc": [
@@ -244,6 +244,7 @@ def find_actor_or_create(actor: str) -> Union[User, Community, None]:
                                         ap_public_url=activity_json['id'],
                                         ap_profile_id=activity_json['id'],
                                         ap_inbox_url=activity_json['endpoints']['sharedInbox'],
+                                        ap_followers_url=activity_json['followers'] if 'followers' in activity_json else None,
                                         ap_preferred_username=activity_json['preferredUsername'],
                                         ap_fetched_at=datetime.utcnow(),
                                         ap_domain=server,

app/community/routes.py

@@ -238,6 +238,12 @@ def add_post(actor):
         form.nsfw.render_kw = {'disabled': True}
     if get_setting('allow_nsfl', False) is False:
         form.nsfl.render_kw = {'disabled': True}
+    if community.nsfw:
+        form.nsfw.data = True
+        form.nsfw.render_kw = {'disabled': True}
+    if community.nsfl:
+        form.nsfl.data = True
+        form.nsfw.render_kw = {'disabled': True}
     images_disabled = 'disabled' if not get_setting('allow_local_image_posts', True) else ''    # bug: this will disable posting of images to *remote* hosts too
 
     form.communities.choices = [(c.id, c.display_name()) for c in current_user.communities()]
@@ -248,11 +254,13 @@ def add_post(actor):
         community.post_count += 1
         community.last_active = datetime.utcnow()
         db.session.commit()
+        post.ap_id = f"https://{current_app.config['SERVER_NAME']}/post/{post.id}"
+        db.session.commit()
 
-        if community.ap_id:  # this is a remote community - send the post to the instance that hosts it
+        if not community.is_local():  # this is a remote community - send the post to the instance that hosts it
             page = {
                 'type': 'Page',
-                'id': f"https://{current_app.config['SERVER_NAME']}/post/{post.id}",
+                'id': post.ap_id,
                 'attributedTo': current_user.ap_profile_id,
                 'to': [
                     community.ap_profile_id,

app/community/util.py

@@ -9,7 +9,7 @@ from pillow_heif import register_heif_opener
 
 from app import db, cache
 from app.constants import POST_TYPE_ARTICLE, POST_TYPE_LINK, POST_TYPE_IMAGE
-from app.models import Community, File, BannedInstances, PostReply, PostVote
+from app.models import Community, File, BannedInstances, PostReply, PostVote, Post
 from app.utils import get_request, gibberish, markdown_to_html, domain_from_url, validate_image
 from sqlalchemy import desc, text
 import os
@@ -133,7 +133,7 @@ def url_to_thumbnail_file(filename) -> File:
                     source_url=filename)
 
 
-def save_post(form, post):
+def save_post(form, post: Post):
     post.nsfw = form.nsfw.data
     post.nsfl = form.nsfl.data
     post.notify_author = form.notify_author.data

app/models.py

@@ -170,6 +170,9 @@ class Community(db.Model):
     def profile_id(self):
         return self.ap_profile_id if self.ap_profile_id else f"https://{current_app.config['SERVER_NAME']}/c/{self.name}"
 
+    def is_local(self):
+        return self.ap_id is None or self.profile_id().startswith('https://' + current_app.config['SERVER_NAME'])
+
 
 user_role = db.Table('user_role',
     db.Column('user_id', db.Integer, db.ForeignKey('user.id')),
@@ -285,12 +288,21 @@ class User(UserMixin, db.Model):
                     return self.cover.source_url
         return ''
 
+    def is_local(self):
+        return self.ap_id is None or self.ap_profile_id.startswith('https://' + current_app.config['SERVER_NAME'])
+
     def link(self) -> str:
-        if self.ap_id is None:
+        if self.is_local():
             return self.user_name
         else:
             return self.ap_id
 
+    def followers_url(self):
+        if self.ap_followers_url:
+            return self.ap_followers_url
+        else:
+            return self.profile_id() + '/followers'
+
     def get_reset_password_token(self, expires_in=600):
         return jwt.encode(
             {'reset_password': self.id, 'exp': time() + expires_in},
@@ -343,6 +355,8 @@ class User(UserMixin, db.Model):
     def profile_id(self):
         return self.ap_profile_id if self.ap_profile_id else f"https://{current_app.config['SERVER_NAME']}/u/{self.user_name}"
 
+
+
     def created_recently(self):
         return self.created and self.created > datetime.utcnow() - timedelta(days=7)
 
@@ -433,6 +447,9 @@ class Post(db.Model):
     domain = db.relationship('Domain', lazy='joined', foreign_keys=[domain_id])
     author = db.relationship('User', lazy='joined', overlaps='posts', foreign_keys=[user_id])
 
+    def is_local(self):
+        return self.ap_id is None or self.ap_id.startswith('https://' + current_app.config['SERVER_NAME'])
+
     @classmethod
     def get_by_ap_id(cls, ap_id):
         return cls.query.filter_by(ap_id=ap_id).first()
@@ -490,6 +507,9 @@ class PostReply(db.Model):
 
     search_vector = db.Column(TSVectorType('body'))
 
+    def is_local(self):
+        return self.ap_id is None or self.ap_id.startswith('https://' + current_app.config['SERVER_NAME'])
+
     @classmethod
     def get_by_ap_id(cls, ap_id):
         return cls.query.filter_by(ap_id=ap_id).first()
@@ -497,6 +517,22 @@ class PostReply(db.Model):
     def profile_id(self):
         return f"https://{current_app.config['SERVER_NAME']}/comment/{self.id}"
 
+    # the ap_id of the parent object, whether it's another PostReply or a Post
+    def in_reply_to(self):
+        if self.parent_id is None:
+            return self.post.ap_id
+        else:
+            parent = PostReply.query.get(self.parent_id)
+            return parent.ap_id
+
+    # the AP profile of the person who wrote the parent object, which could be another PostReply or a Post
+    def to(self):
+        if self.parent_id is None:
+            return self.post.author.profile_id()
+        else:
+            parent = PostReply.query.get(self.parent_id)
+            return parent.author.profile_id()
+
 
 class Domain(db.Model):
     id = db.Column(db.Integer, primary_key=True)

app/post/routes.py

@@ -6,6 +6,7 @@ from flask_babel import _
 from sqlalchemy import or_, desc
 
 from app import db, constants
+from app.activitypub.signature import HttpSignature
 from app.community.util import save_post
 from app.post.forms import NewReplyForm
 from app.community.forms import CreatePostForm
@@ -15,14 +16,15 @@ from app.models import Post, PostReply, \
     PostReplyVote, PostVote, Notification
 from app.post import bp
 from app.utils import get_setting, render_template, allowlist_html, markdown_to_html, validation_required, \
-    shorten_string, markdown_to_text, domain_from_url, validate_image, gibberish
+    shorten_string, markdown_to_text, domain_from_url, validate_image, gibberish, ap_datetime
 
 
-@bp.route('/post/<int:post_id>', methods=['GET', 'POST'])
 def show_post(post_id: int):
     post = Post.query.get_or_404(post_id)
     mods = post.community.moderators()
     is_moderator = current_user.is_authenticated and any(mod.user_id == current_user.id for mod in mods)
+
+    # handle top-level comments/replies
     form = NewReplyForm()
     if current_user.is_authenticated and current_user.verified and form.validate_on_submit():
         reply = PostReply(user_id=current_user.id, post_id=post.id, community_id=post.community.id, body=form.body.data,
@@ -33,8 +35,12 @@ def show_post(post_id: int):
             notification = Notification(title=_('Reply: ') + shorten_string(form.body.data), user_id=post.user_id,
                                         author_id=current_user.id, url=url_for('post.show_post', post_id=post.id))
             db.session.add(notification)
+        post.last_active = post.community.last_active = datetime.utcnow()
+        post.reply_count += 1
+        post.community.post_reply_count += 1
         db.session.add(reply)
         db.session.commit()
+        reply.ap_id = reply.profile_id()
         reply_vote = PostReplyVote(user_id=current_user.id, author_id=current_user.id, post_reply_id=reply.id,
                                    effect=1.0)
         db.session.add(reply_vote)
@@ -42,7 +48,58 @@ def show_post(post_id: int):
         form.body.data = ''
         flash('Your comment has been added.')
         # todo: flush cache
-        # todo: federation
+        # federation
+        if not post.community.is_local():    # this is a remote community, send it to the instance that hosts it
+            reply_json = {
+                'type': 'Note',
+                'id': reply.profile_id(),
+                'attributedTo': current_user.profile_id(),
+                'to': [
+                    'https://www.w3.org/ns/activitystreams#Public'
+                ],
+                'cc': [
+                    post.community.profile_id(),
+                ],
+                'content': reply.body_html,
+                'inReplyTo': post.profile_id(),
+                'mediaType': 'text/html',
+                'source': {
+                    'content': reply.body,
+                    'mediaType': 'text/markdown'
+                },
+                'published': ap_datetime(datetime.utcnow()),
+                'distinguished': False,
+                'audience': post.community.profile_id()
+            }
+            create_json = {
+                'type': 'Create',
+                'actor': current_user.profile_id(),
+                'audience': post.community.profile_id(),
+                'to': [
+                    'https://www.w3.org/ns/activitystreams#Public'
+                ],
+                'cc': [
+                    post.community.ap_profile_id
+                ],
+                'object': reply_json,
+                'id': f"https://{current_app.config['SERVER_NAME']}/activities/create/{gibberish(15)}"
+            }
+
+            try:
+                message = HttpSignature.signed_request(post.community.ap_inbox_url, create_json, current_user.private_key,
+                                                       current_user.ap_profile_id + '#main-key')
+                if message.status_code == 200:
+                    flash('Your reply has been sent to ' + post.community.title)
+                else:
+                    flash('Response status code was not 200', 'warning')
+                    current_app.logger.error('Response code for reply attempt was ' +
+                                             str(message.status_code) + ' ' + message.text)
+            except Exception as ex:
+                flash('Failed to send request to subscribe: ' + str(ex), 'error')
+                current_app.logger.error("Exception while trying to subscribe" + str(ex))
+        else:                       # local community - send it to followers on remote instances
+            ...
+
         return redirect(url_for('post.show_post',
                                 post_id=post_id))  # redirect to current page to avoid refresh resubmitting the form
     else:
@@ -176,31 +233,101 @@ def continue_discussion(post_id, comment_id):
 @login_required
 def add_reply(post_id: int, comment_id: int):
     post = Post.query.get_or_404(post_id)
-    comment = PostReply.query.get_or_404(comment_id)
+    in_reply_to = PostReply.query.get_or_404(comment_id)
     mods = post.community.moderators()
     is_moderator = current_user.is_authenticated and any(mod.user_id == current_user.id for mod in mods)
     form = NewReplyForm()
     if form.validate_on_submit():
-        reply = PostReply(user_id=current_user.id, post_id=post.id, parent_id=comment.id, depth=comment.depth + 1,
+        reply = PostReply(user_id=current_user.id, post_id=post.id, parent_id=in_reply_to.id, depth=in_reply_to.depth + 1,
                           community_id=post.community.id, body=form.body.data,
                           body_html=markdown_to_html(form.body.data), body_html_safe=True,
                           from_bot=current_user.bot, up_votes=1, nsfw=post.nsfw, nsfl=post.nsfl,
                           notify_author=form.notify_author.data)
         db.session.add(reply)
-        if comment.notify_author and current_user.id != comment.user_id:    # todo: check if replier is blocked
-            notification = Notification(title=_('Reply: ') + shorten_string(form.body.data), user_id=comment.user_id,
+        if in_reply_to.notify_author and current_user.id != in_reply_to.user_id and in_reply_to.author.ap_id is None:    # todo: check if replier is blocked
+            notification = Notification(title=_('Reply: ') + shorten_string(form.body.data), user_id=in_reply_to.user_id,
                                         author_id=current_user.id, url=url_for('post.show_post', post_id=post.id))
             db.session.add(notification)
         db.session.commit()
+        reply.ap_id = reply.profile_id()
+        db.session.commit()
         reply_vote = PostReplyVote(user_id=current_user.id, author_id=current_user.id, post_reply_id=reply.id,
                                    effect=1.0)
         db.session.add(reply_vote)
         post.reply_count = post_reply_count(post.id)
+        post.last_active = post.community.last_active = datetime.utcnow()
         db.session.commit()
         form.body.data = ''
         flash('Your comment has been added.')
         # todo: flush cache
-        # todo: federation
+
+        # federation
+        if not post.community.is_local():    # this is a remote community, send it to the instance that hosts it
+            reply_json = {
+                'type': 'Note',
+                'id': reply.profile_id(),
+                'attributedTo': current_user.profile_id(),
+                'to': [
+                    'https://www.w3.org/ns/activitystreams#Public'
+                ],
+                'cc': [
+                    post.community.profile_id(),
+                ],
+                'content': reply.body_html,
+                'inReplyTo': in_reply_to.profile_id(),
+                'mediaType': 'text/html',
+                'source': {
+                    'content': reply.body,
+                    'mediaType': 'text/markdown'
+                },
+                'published': ap_datetime(datetime.utcnow()),
+                'distinguished': False,
+                'audience': post.community.profile_id()
+            }
+            create_json = {
+                'type': 'Create',
+                'actor': current_user.profile_id(),
+                'audience': post.community.profile_id(),
+                'to': [
+                    'https://www.w3.org/ns/activitystreams#Public'
+                ],
+                'cc': [
+                    post.community.ap_profile_id
+                ],
+                'object': reply_json,
+                'id': f"https://{current_app.config['SERVER_NAME']}/activities/create/{gibberish(15)}"
+            }
+            if in_reply_to.notify_author and in_reply_to.author.ap_id is not None:
+                create_json['cc'].append(in_reply_to.author.ap_profile_id)
+                create_json['tag'] = [
+                    {
+                        'href': in_reply_to.author.ap_profile_id,
+                        'name': '@' + in_reply_to.author.ap_id,
+                        'type': 'Mention'
+                    }
+                ]
+                reply_json['cc'].append(in_reply_to.author.ap_profile_id)
+                reply_json['tag'] = [
+                    {
+                        'href': in_reply_to.author.ap_profile_id,
+                        'name': '@' + in_reply_to.author.ap_id,
+                        'type': 'Mention'
+                    }
+                ]
+            try:
+                message = HttpSignature.signed_request(post.community.ap_inbox_url, create_json, current_user.private_key,
+                                                       current_user.ap_profile_id + '#main-key')
+                if message.status_code == 200:
+                    flash('Your reply has been sent to ' + post.community.title)
+                else:
+                    flash('Response status code was not 200', 'warning')
+                    current_app.logger.error('Response code for reply attempt was ' +
+                                             str(message.status_code) + ' ' + message.text)
+            except Exception as ex:
+                flash('Failed to send request to subscribe: ' + str(ex), 'error')
+                current_app.logger.error("Exception while trying to subscribe" + str(ex))
+        else:                       # local community - send it to followers on remote instances
+            ...
         if reply.depth <= constants.THREAD_CUTOFF_DEPTH:
             return redirect(url_for('post.show_post', post_id=post_id, _anchor=f'comment_{reply.parent_id}'))
         else:
@@ -208,7 +335,7 @@ def add_reply(post_id: int, comment_id: int):
     else:
         form.notify_author.data = True
         return render_template('post/add_reply.html', title=_('Discussing %(title)s', title=post.title), post=post,
-                               is_moderator=is_moderator, form=form, comment=comment)
+                               is_moderator=is_moderator, form=form, comment=in_reply_to)
 
 
 @bp.route('/post/<int:post_id>/options', methods=['GET'])