federate incoming posts and post replies

README.md

@@ -2,7 +2,17 @@
 
 A lemmy/kbin clone written in Python with Flask.
 
- - Clean, simple code.
+ - Clean, simple code that is easy to understand and contribute to. No fancy design patterns or algorithms.
  - Easy setup, easy to manage - few dependencies and extra software required.
  - GPL.
+ - First class moderation tools.
+
+## Project goals
+
+To build a ...
+
+
+## Differences from other federated systems
+
+...
 

ROADMAP.md

@@ -11,6 +11,7 @@ The following are the goals for a 1.0 release, good enough for production use. I
 - vote
 - sort posts by hotness algo
 - markdown
+- logging and debugging support
 
 
 ### Activitypub-enabled

app/__init__.py

@@ -12,6 +12,7 @@ from flask_bootstrap import Bootstrap5
 from flask_mail import Mail
 from flask_moment import Moment
 from flask_babel import Babel, lazy_gettext as _l
+from flask_caching import Cache
 from sqlalchemy_searchable import make_searchable
 
 from config import Config
@@ -26,6 +27,7 @@ mail = Mail()
 bootstrap = Bootstrap5()
 moment = Moment()
 babel = Babel()
+cache = Cache(config={'CACHE_TYPE': os.environ.get('CACHE_TYPE'), 'CACHE_DIR': os.environ.get('CACHE_DIR') or '/dev/shm'})
 
 
 def create_app(config_class=Config):
@@ -40,6 +42,7 @@ def create_app(config_class=Config):
     moment.init_app(app)
     make_searchable(db.metadata)
     babel.init_app(app, locale_selector=get_locale)
+    cache.init_app(app)
 
     from app.main import bp as main_bp
     app.register_blueprint(main_bp)

app/activitypub/routes.py

@@ -1,17 +1,20 @@
+import markdown2
 import werkzeug.exceptions
 from sqlalchemy import text
 
 from app import db
 from app.activitypub import bp
-from flask import request, Response, render_template, current_app, abort, jsonify, json
+from flask import request, Response, current_app, abort, jsonify, json
 
 from app.activitypub.signature import HttpSignature
 from app.community.routes import show_community
+from app.constants import POST_TYPE_LINK, POST_TYPE_IMAGE
 from app.models import User, Community, CommunityJoinRequest, CommunityMember, CommunityBan, ActivityPubLog, Post, \
-    PostReply, Instance, PostVote, PostReplyVote
+    PostReply, Instance, PostVote, PostReplyVote, File
 from app.activitypub.util import public_key, users_total, active_half_year, active_month, local_posts, local_comments, \
-    post_to_activity, find_actor_or_create
+    post_to_activity, find_actor_or_create, default_context, instance_blocked, find_reply_parent, find_liked_object
-from app.utils import gibberish, get_setting
+from app.utils import gibberish, get_setting, is_image_url, allowlist_html, html_to_markdown, render_template, \
+    domain_from_url
 
 INBOX = []
 
@@ -105,16 +108,7 @@ def user_profile(actor):
     if user is not None:
         if 'application/ld+json' in request.headers.get('Accept', '') or request.accept_mimetypes.accept_json:
             server = current_app.config['SERVER_NAME']
-            actor_data = {  "@context": [
+            actor_data = {  "@context": default_context(),
-                                "https://www.w3.org/ns/activitystreams",
-                                "https://w3id.org/security/v1",
-                                {
-                                    "manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
-                                    "schema": "http://schema.org#",
-                                    "PropertyValue": "schema:PropertyValue",
-                                    "value": "schema:value"
-                                }
-                            ],
                             "type": "Person",
                             "id": f"https://{server}/u/{actor}",
                             "preferredUsername": actor,
@@ -157,10 +151,7 @@ def community_profile(actor):
     if community is not None:
         if 'application/ld+json' in request.headers.get('Accept', ''):
             server = current_app.config['SERVER_NAME']
-            actor_data = {"@context": [
+            actor_data = {"@context": default_context(),
-                "https://www.w3.org/ns/activitystreams",
-                "https://w3id.org/security/v1"
-                ],
                 "type": "Group",
                 "id": f"https://{server}/c/{actor}",
                 "name": actor.title,
@@ -210,6 +201,7 @@ def shared_inbox():
             request_json = request.get_json(force=True)
         except werkzeug.exceptions.BadRequest as e:
             activity_log.exception_message = 'Unable to parse json body: ' + e.description
+            activity_log.result = 'failure'
             db.session.add(activity_log)
             db.session.commit()
             return
@@ -217,123 +209,205 @@ def shared_inbox():
             if 'id' in request_json:
                 activity_log.activity_id = request_json['id']
 
-        actor = find_actor_or_create(request_json['actor'])
+        actor = find_actor_or_create(request_json['actor']) if 'actor' in request_json else None
         if actor is not None:
             if HttpSignature.verify_request(request, actor.public_key, skip_date=True):
                 if 'type' in request_json:
                     activity_log.activity_type = request_json['type']
-                    if request_json['type'] == 'Announce':
+                    if not instance_blocked(request_json['id']):
-                        if request_json['object']['type'] == 'Like' or request_json['object']['type'] == 'Dislike':
+                        # Announce is new content and votes
-                            activity_log.activity_type = request_json['object']['type']
+                        if request_json['type'] == 'Announce':
-                            vote_effect = 1.0 if request_json['object']['type'] == 'Like' else -1.0
+                            if request_json['object']['type'] == 'Create':
-                            if vote_effect < 0 and get_setting('allow_dislike', True) is False:
+                                activity_log.activity_type = request_json['object']['type']
-                                activity_log.exception_message = 'Dislike ignored because of allow_dislike setting'
+                                user_ap_id = request_json['object']['object']['attributedTo']
-                            else:
+                                community_ap_id = request_json['object']['audience']
-                                user_ap_id = request_json['object']['actor']
+                                community = find_actor_or_create(community_ap_id)
-                                liked_ap_id = request_json['object']['object']
                                 user = find_actor_or_create(user_ap_id)
-                                vote_weight = 1.0
+                                if user and community:
-                                if user.ap_domain:
+                                    object_type = request_json['object']['object']['type']
-                                    instance = Instance.query.filter_by(domain=user.ap_domain).fetch()
+                                    new_content_types = ['Page', 'Article', 'Link', 'Note']
-                                    if instance:
+                                    if object_type in new_content_types:      # create a new post
-                                        vote_weight = instance.vote_weight
+                                        in_reply_to = request_json['object']['object']['inReplyTo'] if 'inReplyTo' in \
-                                liked = find_liked_object(liked_ap_id)
+                                                                                                       request_json['object']['object'] else None
-                                # insert into voted table
+
-                                if isinstance(liked, Post):
+                                        if not in_reply_to:
-                                    existing_vote = PostVote.query.filter_by(user_id=user.id, post_id=liked.id).first()
+                                            post = Post(user_id=user.id, community_id=community.id,
-                                    if existing_vote:
+                                                        title=request_json['object']['object']['name'],
-                                        existing_vote.effect = vote_effect * vote_weight
+                                                        comments_enabled=request_json['object']['object']['commentsEnabled'],
+                                                        sticky=request_json['object']['object']['stickied'] if 'stickied' in request_json['object']['object'] else False,
+                                                        nsfw=request_json['object']['object']['sensitive'],
+                                                        nsfl=request_json['object']['object']['nsfl'] if 'nsfl' in request_json['object']['object'] else False,
+                                                        ap_id=request_json['object']['object']['id'],
+                                                        ap_create_id=request_json['object']['id'],
+                                                        ap_announce_id=request_json['id'],
+                                                        )
+                                            if 'source' in request_json['object']['object'] and \
+                                                    request_json['object']['object']['source']['mediaType'] == 'text/markdown':
+                                                post.body = request_json['object']['object']['source']['content']
+                                                post.body_html = allowlist_html(markdown2.markdown(post.body, safe_mode=True))
+                                            elif 'content' in request_json['object']['object']:
+                                                post.body_html = allowlist_html(request_json['object']['object']['content'])
+                                                post.body = html_to_markdown(post.body_html)
+                                            if 'attachment' in request_json['object']['object'] and \
+                                                    len(request_json['object']['object']['attachment']) > 0 and \
+                                                    'type' in request_json['object']['object']['attachment'][0]:
+                                                if request_json['object']['object']['attachment'][0]['type'] == 'Link':
+                                                    post.url = request_json['object']['object']['attachment'][0]['href']
+                                                    if is_image_url(post.url):
+                                                        post.type = POST_TYPE_IMAGE
+                                                    else:
+                                                        post.type = POST_TYPE_LINK
+                                                    domain = domain_from_url(post.url)
+                                                    if not domain.banned:
+                                                        post.domain_id = domain.id
+                                                    else:
+                                                        post = None
+                                                        activity_log.exception_message = domain.name + ' is blocked by admin'
+                                                        activity_log.result = 'failure'
+                                            if 'image' in request_json['object']['object']:
+                                                image = File(source_url=request_json['object']['object']['image']['url'])
+                                                db.session.add(image)
+                                                post.image = image
+
+                                            if post is not None:
+                                                db.session.add(post)
+                                                db.session.commit()
+                                        else:
+                                            post_id, parent_comment_id, root_id = find_reply_parent(in_reply_to)
+                                            post_reply = PostReply(user_id=user.id, community_id=community.id,
+                                                                   post_id=post_id, parent_id=parent_comment_id,
+                                                                   root_id=root_id,
+                                                                   nsfw=community.nsfw,
+                                                                   nsfl=community.nsfl,
+                                                                   ap_id=request_json['object']['object']['id'],
+                                                                   ap_create_id=request_json['object']['id'],
+                                                                   ap_announce_id=request_json['id'])
+                                            if 'source' in request_json['object']['object'] and \
+                                                    request_json['object']['object']['source'][
+                                                        'mediaType'] == 'text/markdown':
+                                                post_reply.body = request_json['object']['object']['source']['content']
+                                                post_reply.body_html = allowlist_html(markdown2.markdown(post_reply.body, safe_mode=True))
+                                            elif 'content' in request_json['object']['object']:
+                                                post_reply.body_html = allowlist_html(
+                                                    request_json['object']['object']['content'])
+                                                post_reply.body = html_to_markdown(post_reply.body_html)
+
+                                            if post_reply is not None:
+                                                db.session.add(post_reply)
+                                                db.session.commit()
                                     else:
-                                        vote = PostVote(user_id=user.id, author_id=liked.user_id, post_id=liked.id,
+                                        activity_log.exception_message = 'Unacceptable type: ' + object_type
-                                                        effect=vote_effect * vote_weight)
+
-                                        db.session.add(vote)
+                            elif request_json['object']['type'] == 'Like' or request_json['object']['type'] == 'Dislike':
-                                    db.session.commit()
+                                activity_log.activity_type = request_json['object']['type']
-                                    activity_log.result = 'success'
+                                vote_effect = 1.0 if request_json['object']['type'] == 'Like' else -1.0
-                                elif isinstance(liked, PostReply):
+                                if vote_effect < 0 and get_setting('allow_dislike', True) is False:
-                                    existing_vote = PostVote.query.filter_by(user_id=user.id, post_id=liked.id).first()
+                                    activity_log.exception_message = 'Dislike ignored because of allow_dislike setting'
-                                    if existing_vote:
+                                else:
-                                        existing_vote.effect = vote_effect * vote_weight
+                                    user_ap_id = request_json['object']['actor']
+                                    liked_ap_id = request_json['object']['object']
+                                    user = find_actor_or_create(user_ap_id)
+                                    vote_weight = 1.0
+                                    if user.ap_domain:
+                                        instance = Instance.query.filter_by(domain=user.ap_domain).fetch()
+                                        if instance:
+                                            vote_weight = instance.vote_weight
+                                    liked = find_liked_object(liked_ap_id)
+                                    # insert into voted table
+                                    if liked is not None and isinstance(liked, Post):
+                                        existing_vote = PostVote.query.filter_by(user_id=user.id, post_id=liked.id).first()
+                                        if existing_vote:
+                                            existing_vote.effect = vote_effect * vote_weight
+                                        else:
+                                            vote = PostVote(user_id=user.id, author_id=liked.user_id, post_id=liked.id,
+                                                            effect=vote_effect * vote_weight)
+                                            db.session.add(vote)
+                                        db.session.commit()
+                                        activity_log.result = 'success'
+                                    elif liked is not None and isinstance(liked, PostReply):
+                                        existing_vote = PostVote.query.filter_by(user_id=user.id, post_id=liked.id).first()
+                                        if existing_vote:
+                                            existing_vote.effect = vote_effect * vote_weight
+                                        else:
+                                            vote = PostReplyVote(user_id=user.id, author_id=liked.user_id, post_reply_id=liked.id,
+                                                            effect=vote_effect * vote_weight)
+                                            db.session.add(vote)
+                                        db.session.commit()
+                                        activity_log.result = 'success'
                                     else:
-                                        vote = PostReplyVote(user_id=user.id, author_id=liked.user_id, post_reply_id=liked.id,
+                                        activity_log.exception_message = 'Could not detect type of like'
-                                                        effect=vote_effect * vote_weight)
+                                    if activity_log.result == 'success':
-                                        db.session.add(vote)
+                                        ... # todo: recalculate 'hotness' of liked post/reply
-                                    db.session.commit()
+
+                        # Follow: remote user wants to follow one of our communities
+                        elif request_json['type'] == 'Follow':      # Follow is when someone wants to join a community
+                            user_ap_id = request_json['actor']
+                            community_ap_id = request_json['object']
+                            follow_id = request_json['id']
+                            user = find_actor_or_create(user_ap_id)
+                            community = find_actor_or_create(community_ap_id)
+                            if user is not None and community is not None:
+                                # check if user is banned from this community
+                                banned = CommunityBan.query.filter_by(user_id=user.id,
+                                                                      community_id=community.id).first()
+                                if banned is None:
+                                    if not user.subscribed(community):
+                                        member = CommunityMember(user_id=user.id, community_id=community.id)
+                                        db.session.add(member)
+                                        db.session.commit()
+                                    # send accept message to acknowledge the follow
+                                    accept = {
+                                        "@context": default_context(),
+                                        "actor": community.ap_profile_id,
+                                        "to": [
+                                            user.ap_profile_id
+                                        ],
+                                        "object": {
+                                            "actor": user.ap_profile_id,
+                                            "to": None,
+                                            "object": community.ap_profile_id,
+                                            "type": "Follow",
+                                            "id": follow_id
+                                        },
+                                        "type": "Accept",
+                                        "id": f"https://{current_app.config['SERVER_NAME']}/activities/accept/" + gibberish(32)
+                                    }
+                                    try:
+                                        HttpSignature.signed_request(user.ap_inbox_url, accept, community.private_key,
+                                                                     f"https://{current_app.config['SERVER_NAME']}/c/{community.name}#main-key")
+                                    except Exception as e:
+                                        accept_log = ActivityPubLog(direction='out', activity_json=json.dumps(accept),
+                                                                    result='failure', activity_id=accept['id'],
+                                                                    exception_message = 'could not send Accept' + str(e))
+                                        db.session.add(accept_log)
+                                        db.session.commit()
+                                        return
                                     activity_log.result = 'success'
                                 else:
-                                    activity_log.result='failure'
+                                    activity_log.exception_message = 'user is banned from this community'
-                                    activity_log.exception_message = 'Could not detect type of like'
+                        # Accept: remote server is accepting our previous follow request
-                                if activity_log.result == 'success':
+                        elif request_json['type'] == 'Accept':
-                                    ... # todo: recalculate 'hotness' of liked post/reply
+                            if request_json['object']['type'] == 'Follow':
-
+                                community_ap_id = request_json['actor']
-                    # remote user wants to follow one of our communities
+                                user_ap_id = request_json['object']['actor']
-                    elif request_json['type'] == 'Follow':
+                                user = find_actor_or_create(user_ap_id)
-                        user_ap_id = request_json['actor']
+                                community = find_actor_or_create(community_ap_id)
-                        community_ap_id = request_json['object']
+                                join_request = CommunityJoinRequest.query.filter_by(user_id=user.id,
-                        follow_id = request_json['id']
+                                                                                    community_id=community.id).first()
-                        user = find_actor_or_create(user_ap_id)
+                                if join_request:
-                        community = find_actor_or_create(community_ap_id)
-                        if user is not None and community is not None:
-                            # check if user is banned from this community
-                            banned = CommunityBan.query.filter_by(user_id=user.id,
-                                                                  community_id=community.id).first()
-                            if banned is None:
-                                if not user.subscribed(community):
                                     member = CommunityMember(user_id=user.id, community_id=community.id)
                                     db.session.add(member)
                                     db.session.commit()
-                                # send accept message to acknowledge the follow
+                                    activity_log.result = 'success'
-                                accept = {
+                    else:
-                                    "@context": [
+                        activity_log.exception_message = 'Instance banned'
-                                        "https://www.w3.org/ns/activitystreams",
-                                        "https://w3id.org/security/v1",
-                                    ],
-                                    "actor": community.ap_profile_id,
-                                    "to": [
-                                        user.ap_profile_id
-                                    ],
-                                    "object": {
-                                        "actor": user.ap_profile_id,
-                                        "to": None,
-                                        "object": community.ap_profile_id,
-                                        "type": "Follow",
-                                        "id": follow_id
-                                    },
-                                    "type": "Accept",
-                                    "id": f"https://{current_app.config['SERVER_NAME']}/activities/accept/" + gibberish(32)
-                                }
-                                try:
-                                    HttpSignature.signed_request(user.ap_inbox_url, accept, community.private_key,
-                                                                 f"https://{current_app.config['SERVER_NAME']}/c/{community.name}#main-key")
-                                except Exception as e:
-                                    accept_log = ActivityPubLog(direction='out', activity_json=json.dumps(accept),
-                                                                result='failure', activity_id=accept['id'],
-                                                                exception_message = 'could not send Accept' + str(e))
-                                    db.session.add(accept_log)
-                                    db.session.commit()
-                                    return
-                                activity_log.result = 'success'
-                            else:
-                                activity_log.exception_message = 'user is banned from this community'
-                    # remote server is accepting our previous follow request
-                    elif request_json['type'] == 'Accept':
-                        if request_json['object']['type'] == 'Follow':
-                            community_ap_id = request_json['actor']
-                            user_ap_id = request_json['object']['actor']
-                            user = find_actor_or_create(user_ap_id)
-                            community = find_actor_or_create(community_ap_id)
-                            join_request = CommunityJoinRequest.query.filter_by(user_id=user.id,
-                                                                                community_id=community.id).first()
-                            if join_request:
-                                member = CommunityMember(user_id=user.id, community_id=community.id)
-                                db.session.add(member)
-                                db.session.commit()
-                                activity_log.result = 'success'
-
             else:
                 activity_log.exception_message = 'Could not verify signature'
         else:
             activity_log.exception_message = 'Actor could not be found: ' + request_json['actor']
 
+        if activity_log.exception_message is not None:
+            activity_log.result = 'failure'
         db.session.add(activity_log)
         db.session.commit()
 
@@ -346,10 +420,7 @@ def community_outbox(actor):
         posts = community.posts.limit(50).all()
 
         community_data = {
-            "@context": [
+            "@context": default_context(),
-                "https://www.w3.org/ns/activitystreams",
-                "https://w3id.org/security/v1",
-            ],
             "type": "OrderedCollection",
             "id": f"https://{current_app.config['SERVER_NAME']}/c/{actor}/outbox",
             "totalItems": len(posts),

app/activitypub/util.py

@@ -1,22 +1,21 @@
 import json
 import os
 from datetime import datetime
-from typing import Union
+from typing import Union, Tuple
 import markdown2
 from flask import current_app
 from sqlalchemy import text
-from app import db
+from app import db, cache
-from app.models import User, Post, Community, BannedInstances, File
+from app.models import User, Post, Community, BannedInstances, File, PostReply
 import time
 import base64
 import requests
 from cryptography.hazmat.primitives import serialization, hashes
 from cryptography.hazmat.primitives.asymmetric import padding
 from app.constants import *
-import functools
 from urllib.parse import urlparse
 
-from app.utils import get_request
+from app.utils import get_request, allowlist_html
 
 
 def public_key():
@@ -177,8 +176,11 @@ def banned_user_agents():
     return []  # todo: finish this function
 
 
-@functools.lru_cache(maxsize=100)
+@cache.cached(150)
-def instance_blocked(host):
+def instance_blocked(host: str) -> bool:
+    host = host.lower()
+    if 'https://' in host or 'http://' in host:
+        host = urlparse(host).hostname
     instance = BannedInstances.query.filter_by(domain=host.strip()).first()
     return instance is not None
 
@@ -198,7 +200,8 @@ def find_actor_or_create(actor: str) -> Union[User, Community, None]:
         server, address = extract_domain_and_actor(actor)
         if instance_blocked(server):
             return None
-        user = User.query.filter_by(ap_profile_id=actor).first()  # finds users formatted like https://kbin.social/u/tables
+        user = User.query.filter_by(
+            ap_profile_id=actor).first()  # finds users formatted like https://kbin.social/u/tables
         if user is None:
             user = Community.query.filter_by(ap_profile_id=actor).first()
     if user is None:
@@ -301,6 +304,75 @@ def parse_summary(user_json) -> str:
         html_content = markdown2.markdown(markdown_text)
         return html_content
     elif 'summary' in user_json:
-        return user_json['summary']
+        return allowlist_html(user_json['summary'])
     else:
         return ''
+
+
+def default_context():
+    context = [
+        "https://www.w3.org/ns/activitystreams",
+        "https://w3id.org/security/v1",
+    ]
+    if current_app.config['FULL_AP_CONTEXT']:
+        context.append({
+            "lemmy": "https://join-lemmy.org/ns#",
+            "litepub": "http://litepub.social/ns#",
+            "pt": "https://joinpeertube.org/ns#",
+            "sc": "http://schema.org/",
+            "ChatMessage": "litepub:ChatMessage",
+            "commentsEnabled": "pt:commentsEnabled",
+            "sensitive": "as:sensitive",
+            "matrixUserId": "lemmy:matrixUserId",
+            "postingRestrictedToMods": "lemmy:postingRestrictedToMods",
+            "removeData": "lemmy:removeData",
+            "stickied": "lemmy:stickied",
+            "moderators": {
+                "@type": "@id",
+                "@id": "lemmy:moderators"
+            },
+            "expires": "as:endTime",
+            "distinguished": "lemmy:distinguished",
+            "language": "sc:inLanguage",
+            "identifier": "sc:identifier"
+        })
+    return context
+
+
+def find_reply_parent(in_reply_to: str) -> Tuple[int, int, int]:
+    if 'comment' in in_reply_to:
+        parent_comment = PostReply.get_by_ap_id(in_reply_to)
+        parent_comment_id = parent_comment.id
+        post_id = parent_comment.post_id
+        root_id = parent_comment.root_id
+    elif 'post' in in_reply_to:
+        parent_comment_id = None
+        post = Post.get_by_ap_id(in_reply_to)
+        post_id = post.id
+        root_id = None
+    else:
+        parent_comment_id = None
+        root_id = None
+        post_id = None
+        post = Post.get_by_ap_id(in_reply_to)
+        if post:
+            post_id = post.id
+        else:
+            parent_comment = PostReply.get_by_ap_id(in_reply_to)
+            if parent_comment:
+                parent_comment_id = parent_comment.id
+                post_id = parent_comment.post_id
+                root_id = parent_comment.root_id
+
+    return post_id, parent_comment_id, root_id
+
+
+def find_liked_object(ap_id) -> Union[Post, PostReply, None]:
+    post = Post.get_by_ap_id(ap_id)
+    if post:
+        return post
+    else:
+        post_reply = PostReply.get_by_ap_id(ap_id)
+        if post_reply:
+            return post_reply
+    return None

app/admin/routes.py

@@ -0,0 +1 @@
+from app.utils import render_template

app/auth/routes.py

@@ -1,5 +1,5 @@
 from datetime import date, datetime, timedelta
-from flask import render_template, redirect, url_for, flash, request, make_response, session, Markup, current_app
+from flask import redirect, url_for, flash, request, make_response, session, Markup, current_app
 from werkzeug.urls import url_parse
 from flask_login import login_user, logout_user, current_user
 from flask_babel import _
@@ -10,6 +10,7 @@ from app.auth.util import random_token
 from app.models import User
 from app.auth.email import send_password_reset_email, send_welcome_email, send_verification_email
 from app.activitypub.signature import RsaKeys
+from app.utils import render_template
 
 
 @bp.route('/login', methods=['GET', 'POST'])

app/community/routes.py

@@ -1,5 +1,5 @@
 from datetime import date, datetime, timedelta
-from flask import render_template, redirect, url_for, flash, request, make_response, session, Markup, current_app, abort
+from flask import redirect, url_for, flash, request, make_response, session, Markup, current_app, abort
 from flask_login import login_user, logout_user, current_user
 from flask_babel import _
 from app import db
@@ -9,7 +9,7 @@ from app.community.util import search_for_community, community_url_exists
 from app.constants import SUBSCRIPTION_MEMBER, SUBSCRIPTION_OWNER
 from app.models import User, Community, CommunityMember, CommunityJoinRequest, CommunityBan
 from app.community import bp
-from app.utils import get_setting
+from app.utils import get_setting, render_template
 from sqlalchemy import or_
 
 

app/main/routes.py

@@ -2,12 +2,13 @@ from datetime import datetime
 
 from app import db
 from app.main import bp
-from flask import g, jsonify, render_template, flash, request
+from flask import g, jsonify, flash, request
 from flask_moment import moment
 from flask_login import current_user
 from flask_babel import _, get_locale
 from sqlalchemy import select
 from sqlalchemy_searchable import search
+from app.utils import render_template
 
 from app.models import Community, CommunityMember
 

app/models.py

@@ -231,6 +231,7 @@ class Post(db.Model):
     body = db.Column(db.Text)
     body_html = db.Column(db.Text)
     type = db.Column(db.Integer)
+    comments_enabled = db.Column(db.Boolean, default=True)
     has_embed = db.Column(db.Boolean, default=False)
     reply_count = db.Column(db.Integer, default=0)
     score = db.Column(db.Integer, default=0, index=True)
@@ -256,6 +257,10 @@ class Post(db.Model):
 
     image = db.relationship(File, foreign_keys=[image_id], cascade="all, delete")
 
+    @classmethod
+    def get_by_ap_id(cls, ap_id):
+        return cls.query.filter_by(ap_id=ap_id).first()
+
 
 class PostReply(db.Model):
     id = db.Column(db.Integer, primary_key=True)
@@ -267,6 +272,7 @@ class PostReply(db.Model):
     root_id = db.Column(db.Integer)
     body = db.Column(db.Text)
     body_html = db.Column(db.Text)
+    body_html_safe = db.Column(db.Boolean, default=False)
     score = db.Column(db.Integer, default=0, index=True)
     nsfw = db.Column(db.Boolean, default=False)
     nsfl = db.Column(db.Boolean, default=False)
@@ -280,15 +286,21 @@ class PostReply(db.Model):
     edited_at = db.Column(db.DateTime)
 
     ap_id = db.Column(db.String(255), index=True)
+    ap_create_id = db.Column(db.String(100))
+    ap_announce_id = db.Column(db.String(100))
 
     search_vector = db.Column(TSVectorType('body'))
 
+    @classmethod
+    def get_by_ap_id(cls, ap_id):
+        return cls.query.filter_by(ap_id=ap_id).first()
+
 
 class Domain(db.Model):
     id = db.Column(db.Integer, primary_key=True)
     name = db.Column(db.String(255), index=True)
     post_count = db.Column(db.Integer, default=0)
-    banned = db.Column(db.Boolean, default=False, index=True)
+    banned = db.Column(db.Boolean, default=False, index=True) # Domains can be banned site-wide (by admin) or DomainBlock'ed by users
 
 
 class DomainBlock(db.Model):

app/templates/list_communities.html

@@ -49,10 +49,12 @@
                         <td>{{ community.post_count }}</td>
                         <td>{{ community.post_reply_count }}</td>
                         <td>{{ moment(community.last_active).fromNow(refresh=True) }}</td>
-                        <td>{% if current_user.subscribed(community) %}
+                        <td>{% if current_user.is_authenticated %}
-                                <a class="btn btn-primary btn-sm" href="/community/{{ community.link() }}/unsubscribe">Unsubscribe</a>
+                                {% if current_user.subscribed(community) %}
-                            {% else %}
+                                    <a class="btn btn-primary btn-sm" href="/community/{{ community.link() }}/unsubscribe">Unsubscribe</a>
-                                <a class="btn btn-primary btn-sm" href="/community/{{ community.link() }}/subscribe">Subscribe</a>
+                                {% else %}
+                                    <a class="btn btn-primary btn-sm" href="/community/{{ community.link() }}/subscribe">Subscribe</a>
+                                {% endif %}
                             {% endif %}
                         </td>
                     </tr>

app/utils.py

@@ -1,13 +1,26 @@
 import functools
 import random
+from urllib.parse import urlparse
+
+import flask
+from bs4 import BeautifulSoup
+import html as html_module
 import requests
 import os
 from flask import current_app, json
-from app import db
+from app import db, cache
-from app.models import Settings
+from app.models import Settings, Domain, Instance, BannedInstances
+
+
+# Flask's render_template function, with support for themes added
+def render_template(template_name: str, **context) -> str:
+    theme = get_setting('theme', '')
+    if theme != '':
+        return flask.render_template(f'themes/{theme}/{template_name}', **context)
+    else:
+        return flask.render_template(template_name, **context)
 
 
-# ----------------------------------------------------------------------
 # Jinja: when a file was modified. Useful for cache-busting
 def getmtime(filename):
     return os.path.getmtime('static/' + filename)
@@ -28,8 +41,8 @@ def get_request(uri, params=None, headers=None) -> requests.Response:
     return response
 
 
-# saves an arbitrary object into a persistent key-value store. Possibly redis would be faster than using the DB
+# saves an arbitrary object into a persistent key-value store. cached.
-@functools.lru_cache(maxsize=100)
+@cache.cached(timeout=50)
 def get_setting(name: str, default=None):
     setting = Settings.query.filter_by(name=name).first()
     if setting is None:
@@ -46,7 +59,7 @@ def set_setting(name: str, value):
     else:
         setting.value = json.dumps(value)
     db.session.commit()
-    get_setting.cache_clear()
+    cache.delete_memoized(get_setting)
 
 
 # Return the contents of a file as a string. Inspired by PHP's function of the same name.
@@ -61,3 +74,73 @@ random_chars = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
 
 def gibberish(length: int = 10) -> str:
     return "".join([random.choice(random_chars) for x in range(length)])
+
+
+def is_image_url(url):
+    parsed_url = urlparse(url)
+    path = parsed_url.path.lower()
+    common_image_extensions = ['.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.webp']
+    return any(path.endswith(extension) for extension in common_image_extensions)
+
+
+# sanitise HTML using an allow list
+def allowlist_html(html: str) -> str:
+    allowed_tags = ['p', 'strong', 'a', 'ul', 'ol', 'li', 'em', 'blockquote', 'cite', 'br', 'h3', 'h4', 'h5']
+    # Parse the HTML using BeautifulSoup
+    soup = BeautifulSoup(html, 'html.parser')
+
+    # Find all tags in the parsed HTML
+    for tag in soup.find_all():
+        # If the tag is not in the allowed_tags list, remove it and its contents
+        if tag.name not in allowed_tags:
+            tag.extract()
+        else:
+            # Filter and sanitize attributes
+            for attr in list(tag.attrs):
+                if attr not in ['href', 'src']:  # Add allowed attributes here
+                    del tag[attr]
+
+    # Encode the HTML to prevent script execution
+    return html_module.escape(str(soup))
+
+
+# convert basic HTML to Markdown
+def html_to_markdown(html: str) -> str:
+    soup = BeautifulSoup(html, 'html.parser')
+    return html_to_markdown_worker(soup)
+
+
+def html_to_markdown_worker(element, indent_level=0):
+    formatted_text = ''
+    for item in element.contents:
+        if isinstance(item, str):
+            formatted_text += item
+        elif item.name == 'p':
+            formatted_text += '\n\n'
+        elif item.name == 'br':
+            formatted_text += '  \n'  # Double space at the end for line break
+        elif item.name == 'strong':
+            formatted_text += '**' + html_to_markdown_worker(item) + '**'
+        elif item.name == 'ul':
+            formatted_text += '\n'
+            formatted_text += html_to_markdown_worker(item, indent_level + 1)
+            formatted_text += '\n'
+        elif item.name == 'ol':
+            formatted_text += '\n'
+            formatted_text += html_to_markdown_worker(item, indent_level + 1)
+            formatted_text += '\n'
+        elif item.name == 'li':
+            bullet = '-' if item.find_parent(['ul', 'ol']) and item.find_previous_sibling() is None else ''
+            formatted_text += '  ' * indent_level + bullet + ' ' + html_to_markdown_worker(item).strip() + '\n'
+        elif item.name == 'blockquote':
+            formatted_text += '  ' * indent_level + '> ' + html_to_markdown_worker(item).strip() + '\n'
+        elif item.name == 'code':
+            formatted_text += '`' + html_to_markdown_worker(item) + '`'
+    return formatted_text
+
+
+def domain_from_url(url: str) -> Domain:
+    parsed_url = urlparse(url)
+    domain = Domain.query.filter_by(name=parsed_url.hostname.lower()).first()
+    return domain
+

config.py

@@ -21,3 +21,9 @@ class Config(object):
     RECAPTCHA3_PRIVATE_KEY = os.environ.get("RECAPTCHA3_PRIVATE_KEY")
     MODE = os.environ.get('MODE') or 'development'
     LANGUAGES = ['en']
+    FULL_AP_CONTEXT = os.environ.get('FULL_AP_CONTEXT') or True
+    CACHE_TYPE = os.environ.get('CACHE_TYPE') or 'FileSystemCache'
+    CACHE_DIR = os.environ.get('CACHE_DIR') or '/dev/shm/pyfedi'
+    CACHE_DEFAULT_TIMEOUT = 300
+    CACHE_THRESHOLD = 1000
+    CACHE_KEY_PREFIX = 'pyfedi'

requirements.txt

@@ -20,3 +20,5 @@ arrow==1.2.3
 pyld==2.0.3
 boto3==1.28.35
 markdown2==2.4.10
+beautifulsoup4==4.12.2
+flask-caching==2.0.2