backfill old posts when a remote community is added

app/activitypub/util.py

@@ -4,7 +4,7 @@ from datetime import datetime
 from typing import Union, Tuple
 from flask import current_app, request
 from sqlalchemy import text
-from app import db, cache
+from app import db, cache, constants
 from app.models import User, Post, Community, BannedInstances, File, PostReply, AllowedInstances, Instance, utcnow, Site
 import time
 import base64
@@ -14,7 +14,8 @@ from cryptography.hazmat.primitives.asymmetric import padding
 from app.constants import *
 from urllib.parse import urlparse
 
-from app.utils import get_request, allowlist_html, html_to_markdown, get_setting, ap_datetime
+from app.utils import get_request, allowlist_html, html_to_markdown, get_setting, ap_datetime, markdown_to_html, \
+    is_image_url, domain_from_url
 
 
 def public_key():
@@ -196,6 +197,8 @@ def instance_allowed(host: str) -> bool:
 def find_actor_or_create(actor: str) -> Union[User, Community, None]:
     user = None
     # actor parameter must be formatted as https://server/u/actor or https://server/c/actor
+
+    # Initially, check if the user exists in the local DB already
     if current_app.config['SERVER_NAME'] + '/c/' in actor:
         return Community.query.filter_by(
             ap_profile_id=actor).first()  # finds communities formatted like https://localhost/c/*
@@ -218,86 +221,36 @@ def find_actor_or_create(actor: str) -> Union[User, Community, None]:
             return None
         if user is None:
             user = Community.query.filter_by(ap_profile_id=actor).first()
-    if user is None:
+
-        # retrieve user details via webfinger, etc
+    if user is not None:
-        # todo: try, except block around every get_request
-        webfinger_data = get_request(f"https://{server}/.well-known/webfinger",
-                                     params={'resource': f"acct:{address}@{server}"})
-        if webfinger_data.status_code == 200:
-            webfinger_json = webfinger_data.json()
-            webfinger_data.close()
-            for links in webfinger_json['links']:
-                if 'rel' in links and links['rel'] == 'self':  # this contains the URL of the activitypub profile
-                    type = links['type'] if 'type' in links else 'application/activity+json'
-                    # retrieve the activitypub profile
-                    actor_data = get_request(links['href'], headers={'Accept': type})
-                    # to see the structure of the json contained in actor_data, do a GET to https://lemmy.world/c/technology with header Accept: application/activity+json
-                    if actor_data.status_code == 200:
-                        activity_json = actor_data.json()
-                        actor_data.close()
-                        if activity_json['type'] == 'Person':
-                            user = User(user_name=activity_json['preferredUsername'],
-                                        email=f"{address}@{server}",
-                                        about=parse_summary(activity_json),
-                                        created=activity_json['published'],
-                                        ap_id=f"{address}@{server}",
-                                        ap_public_url=activity_json['id'],
-                                        ap_profile_id=activity_json['id'],
-                                        ap_inbox_url=activity_json['endpoints']['sharedInbox'],
-                                        ap_followers_url=activity_json['followers'] if 'followers' in activity_json else None,
-                                        ap_preferred_username=activity_json['preferredUsername'],
-                                        ap_fetched_at=utcnow(),
-                                        ap_domain=server,
-                                        public_key=activity_json['publicKey']['publicKeyPem'],
-                                        # language=community_json['language'][0]['identifier'] # todo: language
-                                        )
-                            if 'icon' in activity_json:
-                                # todo: retrieve icon, save to disk, save more complete File record
-                                avatar = File(source_url=activity_json['icon']['url'])
-                                user.avatar = avatar
-                                db.session.add(avatar)
-                            if 'image' in activity_json:
-                                # todo: retrieve image, save to disk, save more complete File record
-                                cover = File(source_url=activity_json['image']['url'])
-                                user.cover = cover
-                                db.session.add(cover)
-                            db.session.add(user)
-                            db.session.commit()
-                            return user
-                        elif activity_json['type'] == 'Group':
-                            community = Community(name=activity_json['preferredUsername'],
-                                                  title=activity_json['name'],
-                                                  description=activity_json['summary'],
-                                                  nsfw=activity_json['sensitive'],
-                                                  restricted_to_mods=activity_json['postingRestrictedToMods'],
-                                                  created_at=activity_json['published'],
-                                                  last_active=activity_json['updated'],
-                                                  ap_id=f"{address[1:]}",
-                                                  ap_public_url=activity_json['id'],
-                                                  ap_profile_id=activity_json['id'],
-                                                  ap_followers_url=activity_json['followers'],
-                                                  ap_inbox_url=activity_json['endpoints']['sharedInbox'],
-                                                  ap_fetched_at=utcnow(),
-                                                  ap_domain=server,
-                                                  public_key=activity_json['publicKey']['publicKeyPem'],
-                                                  # language=community_json['language'][0]['identifier'] # todo: language
-                                                  )
-                            if 'icon' in activity_json:
-                                # todo: retrieve icon, save to disk, save more complete File record
-                                icon = File(source_url=activity_json['icon']['url'])
-                                community.icon = icon
-                                db.session.add(icon)
-                            if 'image' in activity_json:
-                                # todo: retrieve image, save to disk, save more complete File record
-                                image = File(source_url=activity_json['image']['url'])
-                                community.image = image
-                                db.session.add(image)
-                            db.session.add(community)
-                            db.session.commit()
-                            return community
-        return None
-    else:
         return user
+    else:   # User does not exist in the DB, it's going to need to be created from it's remote home instance
+        if actor.startswith('https://'):
+            actor_data = get_request(actor, headers={'Accept': 'application/activity+json'})
+            if actor_data.status_code == 200:
+                actor_json = actor_data.json()
+                actor_data.close()
+                return actor_json_to_model(actor_json, address, server)
+        else:
+            # retrieve user details via webfinger, etc
+            # todo: try, except block around every get_request
+            webfinger_data = get_request(f"https://{server}/.well-known/webfinger",
+                                         params={'resource': f"acct:{address}@{server}"})
+            if webfinger_data.status_code == 200:
+                webfinger_json = webfinger_data.json()
+                webfinger_data.close()
+                for links in webfinger_json['links']:
+                    if 'rel' in links and links['rel'] == 'self':  # this contains the URL of the activitypub profile
+                        type = links['type'] if 'type' in links else 'application/activity+json'
+                        # retrieve the activitypub profile
+                        print('****', links['href'])
+                        actor_data = get_request(links['href'], headers={'Accept': type})
+                        # to see the structure of the json contained in actor_data, do a GET to https://lemmy.world/c/technology with header Accept: application/activity+json
+                        if actor_data.status_code == 200:
+                            actor_json = actor_data.json()
+                            actor_data.close()
+                            return actor_json_to_model(actor_json, address, server)
+    return None
 
 
 def extract_domain_and_actor(url_string: str):
@@ -313,6 +266,133 @@ def extract_domain_and_actor(url_string: str):
     return server_domain, actor
 
 
+def actor_json_to_model(activity_json, address, server):
+    if activity_json['type'] == 'Person':
+        user = User(user_name=activity_json['preferredUsername'],
+                    email=f"{address}@{server}",
+                    about=parse_summary(activity_json),
+                    created=activity_json['published'] if 'published' in activity_json else utcnow(),
+                    ap_id=f"{address}@{server}",
+                    ap_public_url=activity_json['id'],
+                    ap_profile_id=activity_json['id'],
+                    ap_inbox_url=activity_json['endpoints']['sharedInbox'],
+                    ap_followers_url=activity_json['followers'] if 'followers' in activity_json else None,
+                    ap_preferred_username=activity_json['preferredUsername'],
+                    ap_fetched_at=utcnow(),
+                    ap_domain=server,
+                    public_key=activity_json['publicKey']['publicKeyPem'],
+                    instance_id=find_instance_id(server)
+                    # language=community_json['language'][0]['identifier'] # todo: language
+                    )
+        if 'icon' in activity_json:
+            # todo: retrieve icon, save to disk, save more complete File record
+            avatar = File(source_url=activity_json['icon']['url'])
+            user.avatar = avatar
+            db.session.add(avatar)
+        if 'image' in activity_json:
+            # todo: retrieve image, save to disk, save more complete File record
+            cover = File(source_url=activity_json['image']['url'])
+            user.cover = cover
+            db.session.add(cover)
+        db.session.add(user)
+        db.session.commit()
+        return user
+    elif activity_json['type'] == 'Group':
+        if 'attributedTo' in activity_json:  # lemmy and mbin
+            mods_url = activity_json['attributedTo']
+        elif 'moderators' in activity_json:  # kbin
+            mods_url = activity_json['moderators']
+        else:
+            mods_url = None
+        community = Community(name=activity_json['preferredUsername'],
+                              title=activity_json['name'],
+                              description=activity_json['summary'] if 'summary' in activity_json else '',
+                              rules=activity_json['rules'] if 'rules' in activity_json else '',
+                              rules_html=markdown_to_html(activity_json['rules'] if 'rules' in activity_json else ''),
+                              nsfw=activity_json['sensitive'],
+                              restricted_to_mods=activity_json['postingRestrictedToMods'],
+                              created_at=activity_json['published'] if 'published' in activity_json else utcnow(),
+                              last_active=activity_json['updated'] if 'updated' in activity_json else utcnow(),
+                              ap_id=f"{address[1:]}",
+                              ap_public_url=activity_json['id'],
+                              ap_profile_id=activity_json['id'],
+                              ap_followers_url=activity_json['followers'],
+                              ap_inbox_url=activity_json['endpoints']['sharedInbox'],
+                              ap_moderators_url=mods_url,
+                              ap_fetched_at=utcnow(),
+                              ap_domain=server,
+                              public_key=activity_json['publicKey']['publicKeyPem'],
+                              # language=community_json['language'][0]['identifier'] # todo: language
+                              instance_id=find_instance_id(server),
+                              low_quality='memes' in activity_json['preferredUsername']
+                              )
+        # parse markdown and overwrite html field with result
+        if 'source' in activity_json and \
+                activity_json['source']['mediaType'] == 'text/markdown':
+            community.description = activity_json['source']['content']
+            community.description_html = markdown_to_html(community.description)
+        elif 'content' in activity_json:
+            community.description_html = allowlist_html(activity_json['content'])
+            community.description = html_to_markdown(community.description_html)
+        if 'icon' in activity_json:
+            # todo: retrieve icon, save to disk, save more complete File record
+            icon = File(source_url=activity_json['icon']['url'])
+            community.icon = icon
+            db.session.add(icon)
+        if 'image' in activity_json:
+            # todo: retrieve image, save to disk, save more complete File record
+            image = File(source_url=activity_json['image']['url'])
+            community.image = image
+            db.session.add(image)
+        db.session.add(community)
+        db.session.commit()
+        return community
+
+
+def post_json_to_model(post_json, user, community) -> Post:
+    post = Post(user_id=user.id, community_id=community.id,
+                title=post_json['name'],
+                comments_enabled=post_json['commentsEnabled'],
+                sticky=post_json['stickied'] if 'stickied' in post_json else False,
+                nsfw=post_json['sensitive'],
+                nsfl=post_json['nsfl'] if 'nsfl' in post_json else False,
+                ap_id=post_json['id'],
+                type=constants.POST_TYPE_ARTICLE,
+                posted_at=post_json['published'],
+                last_active=post_json['published'],
+                )
+    if 'source' in post_json and \
+            post_json['source']['mediaType'] == 'text/markdown':
+        post.body = post_json['source']['content']
+        post.body_html = markdown_to_html(post.body)
+    elif 'content' in post_json:
+        post.body_html = allowlist_html(post_json['content'])
+        post.body = html_to_markdown(post.body_html)
+    if 'attachment' in post_json and \
+            len(post_json['attachment']) > 0 and \
+            'type' in post_json['attachment'][0]:
+        if post_json['attachment'][0]['type'] == 'Link':
+            post.url = post_json['attachment'][0]['href']
+            if is_image_url(post.url):
+                post.type = POST_TYPE_IMAGE
+            else:
+                post.type = POST_TYPE_LINK
+            domain = domain_from_url(post.url)
+            if not domain.banned:
+                post.domain_id = domain.id
+            else:
+                post = None
+    if 'image' in post_json:
+        image = File(source_url=post_json['image']['url'])
+        db.session.add(image)
+        post.image = image
+
+    if post is not None:
+        db.session.add(post)
+        community.post_count += 1
+        db.session.commit()
+    return post
+
 # create a summary from markdown if present, otherwise use html if available
 def parse_summary(user_json) -> str:
     if 'source' in user_json and user_json['source'].get('mediaType') == 'text/markdown':
@@ -395,6 +475,40 @@ def find_liked_object(ap_id) -> Union[Post, PostReply, None]:
     return None
 
 
+def find_instance_id(server):
+    server = server.strip()
+    instance = Instance.query.filter_by(domain=server).first()
+    if instance:
+        return instance.id
+    else:
+        instance_data = get_request(f"https://{server}", headers={'Accept': 'application/activity+json'})
+        if instance_data.status_code == 200:
+            try:
+                instance_json = instance_data.json()
+                instance_data.close()
+            except requests.exceptions.JSONDecodeError as ex:
+                instance_json = {}
+            if 'type' in instance_json and instance_json['type'] == 'Application':
+                if instance_json['name'].lower() == 'kbin':
+                    software = 'Kbin'
+                elif instance_json['name'].lower() == 'mbin':
+                    software = 'Mbin'
+                else:
+                    software = 'Lemmy'
+                new_instance = Instance(domain=server,
+                                        inbox=instance_json['inbox'],
+                                        outbox=instance_json['outbox'],
+                                        software=software,
+                                        created_at=instance_json['published'] if 'published' in instance_json else utcnow()
+                                        )
+            else:
+                new_instance = Instance(domain=server, software='unknown', created_at=utcnow())
+            db.session.add(new_instance)
+            db.session.commit()
+            return new_instance.id
+    return None
+
+
 # alter the effect of upvotes based on their instance. Default to 1.0
 @cache.memoize(timeout=50)
 def instance_weight(domain):

app/cli.py

@@ -83,6 +83,7 @@ def register(app):
             db.session.add(Interest(name='🛠 Programming', communities=parse_communities(interests, 'programming')))
             db.session.add(Interest(name='🖥️ Tech', communities=parse_communities(interests, 'tech')))
             db.session.add(Interest(name='🤗 Mental Health', communities=parse_communities(interests, 'mental health')))
+            db.session.add(Interest(name='💊 Health', communities=parse_communities(interests, 'health')))
 
             # Load initial domain block list
             block_list = retrieve_block_list()

app/community/forms.py

@@ -97,3 +97,7 @@ class ReportCommunityForm(FlaskForm):
     description = StringField(_l('More info'))
     report_remote = BooleanField('Also send report to originating instance')
     submit = SubmitField(_l('Report'))
+
+
+class DeleteCommunityForm(FlaskForm):
+    submit = SubmitField(_l('Delete community'))

app/community/routes.py

@@ -6,7 +6,8 @@ from sqlalchemy import or_, desc
 from app import db, constants, cache
 from app.activitypub.signature import RsaKeys, HttpSignature
 from app.activitypub.util import default_context
-from app.community.forms import SearchRemoteCommunity, AddLocalCommunity, CreatePostForm, ReportCommunityForm
+from app.community.forms import SearchRemoteCommunity, AddLocalCommunity, CreatePostForm, ReportCommunityForm, \
+    DeleteCommunityForm
 from app.community.util import search_for_community, community_url_exists, actor_to_community, \
     ensure_directory_exists, opengraph_parse, url_to_thumbnail_file, save_post, save_icon_file, save_banner_file
 from app.constants import SUBSCRIPTION_MEMBER, SUBSCRIPTION_OWNER, POST_TYPE_LINK, POST_TYPE_ARTICLE, POST_TYPE_IMAGE, \
@@ -37,7 +38,7 @@ def add_local():
                               rules=form.rules.data, nsfw=form.nsfw.data, private_key=private_key,
                               public_key=public_key,
                               ap_profile_id='https://' + current_app.config['SERVER_NAME'] + '/c/' + form.url.data,
-                              subscriptions_count=1, instance_id=1)
+                              subscriptions_count=1, instance_id=1, low_quality='memes' in form.url.data)
         icon_file = request.files['icon_file']
         if icon_file and icon_file.filename != '':
             file = save_icon_file(icon_file)
@@ -413,6 +414,25 @@ def community_report(community_id: int):
     return render_template('community/community_report.html', title=_('Report community'), form=form, community=community)
 
 
+@login_required
+@bp.route('/community/<int:community_id>/delete', methods=['GET', 'POST'])
+def community_delete(community_id: int):
+    community = Community.query.get_or_404(community_id)
+    if community.is_owner() or current_user.is_admin():
+        form = DeleteCommunityForm()
+        if form.validate_on_submit():
+            community.delete_dependencies()
+            db.session.delete(community)
+            db.session.commit()
+            flash(_('Community deleted'))
+            return redirect('/communities')
+
+        return render_template('community/community_delete.html', title=_('Delete community'), form=form,
+                               community=community)
+    else:
+        abort(401)
+
+
 @login_required
 @bp.route('/community/<int:community_id>/block_instance', methods=['GET', 'POST'])
 def community_block_instance(community_id: int):

app/community/util.py

@@ -1,16 +1,19 @@
 from datetime import datetime
+from threading import Thread
+from time import sleep
 from typing import List
-
 import requests
 from PIL import Image, ImageOps
-from flask import request, abort, g
+from flask import request, abort, g, current_app
 from flask_login import current_user
 from pillow_heif import register_heif_opener
 
 from app import db, cache
+from app.activitypub.util import find_actor_or_create, actor_json_to_model, post_json_to_model
 from app.constants import POST_TYPE_ARTICLE, POST_TYPE_LINK, POST_TYPE_IMAGE
-from app.models import Community, File, BannedInstances, PostReply, PostVote, Post, utcnow
+from app.models import Community, File, BannedInstances, PostReply, PostVote, Post, utcnow, CommunityMember
-from app.utils import get_request, gibberish, markdown_to_html, domain_from_url, validate_image
+from app.utils import get_request, gibberish, markdown_to_html, domain_from_url, validate_image, allowlist_html, \
+    html_to_markdown, is_image_url
 from sqlalchemy import desc, text
 import os
 from opengraph_parse import parse_page
@@ -18,6 +21,7 @@ from opengraph_parse import parse_page
 
 allowed_extensions = ['.gif', '.jpg', '.jpeg', '.png', '.webp', '.heic']
 
+
 def search_for_community(address: str):
     if address.startswith('!'):
         name, server = address[1:].split('@')
@@ -45,42 +49,62 @@ def search_for_community(address: str):
                     # to see the structure of the json contained in community_data, do a GET to https://lemmy.world/c/technology with header Accept: application/activity+json
                     if community_data.status_code == 200:
                         community_json = community_data.json()
+                        community_data.close()
                         if community_json['type'] == 'Group':
-                            community = Community(name=community_json['preferredUsername'],
+                            community = actor_json_to_model(community_json, address, server)
-                                                  title=community_json['name'],
+                            thr = Thread(target=retrieve_mods_and_backfill_thread, args=[community, current_app._get_current_object()])
-                                                  description=community_json['summary'],
+                            thr.start()
-                                                  nsfw=community_json['sensitive'],
-                                                  restricted_to_mods=community_json['postingRestrictedToMods'],
-                                                  created_at=community_json['published'],
-                                                  last_active=community_json['updated'],
-                                                  ap_id=f"{address[1:]}",
-                                                  ap_public_url=community_json['id'],
-                                                  ap_profile_id=community_json['id'],
-                                                  ap_followers_url=community_json['followers'],
-                                                  ap_inbox_url=community_json['endpoints']['sharedInbox'],
-                                                  ap_moderators_url=community_json['attributedTo'] if 'attributedTo' in community_json else None,
-                                                  ap_fetched_at=utcnow(),
-                                                  ap_domain=server,
-                                                  public_key=community_json['publicKey']['publicKeyPem'],
-                                                  # language=community_json['language'][0]['identifier'] # todo: language
-                                                  # todo: set instance_id
-                                                  )
-                            if 'icon' in community_json:
-                                # todo: retrieve icon, save to disk, save more complete File record
-                                icon = File(source_url=community_json['icon']['url'])
-                                community.icon = icon
-                                db.session.add(icon)
-                            if 'image' in community_json:
-                                # todo: retrieve image, save to disk, save more complete File record
-                                image = File(source_url=community_json['image']['url'])
-                                community.image = image
-                                db.session.add(image)
-                            db.session.add(community)
-                            db.session.commit()
                             return community
         return None
 
 
+def retrieve_mods_and_backfill_thread(community: Community, app):
+    with app.app_context():
+        if community.ap_moderators_url:
+            mods_request = get_request(community.ap_moderators_url, headers={'Accept': 'application/activity+json'})
+            if mods_request.status_code == 200:
+                mods_data = mods_request.json()
+                mods_request.close()
+                if mods_data and mods_data['type'] == 'OrderedCollection' and 'orderedItems' in mods_data:
+                    for actor in mods_data['orderedItems']:
+                        sleep(0.5)
+                        user = find_actor_or_create(actor)
+                        if user:
+                            existing_membership = CommunityMember.query.filter_by(community_id=community.id, user_id=user.id).first()
+                            if existing_membership:
+                                existing_membership.is_moderator = True
+                            else:
+                                new_membership = CommunityMember(community_id=community.id, user_id=user.id, is_moderator=True)
+                                db.session.add(new_membership)
+                    db.session.commit()
+
+        # only backfill nsfw if nsfw communities are allowed
+        if (community.nsfw and not g.site.enable_nsfw) or (community.nsfl and not g.site.enable_nsfl):
+            return
+
+        # download 50 old posts
+        if community.ap_public_url:
+            outbox_request = get_request(community.ap_public_url + '/outbox', headers={'Accept': 'application/activity+json'})
+            if outbox_request.status_code == 200:
+                outbox_data = outbox_request.json()
+                outbox_request.close()
+                if outbox_data['type'] == 'OrderedCollection' and 'orderedItems' in outbox_data:
+                    activities_processed = 0
+                    for activity in outbox_data['orderedItems']:
+                        user = find_actor_or_create(activity['object']['actor'])
+                        if user:
+                            post = post_json_to_model(activity['object']['object'], user, community)
+                            post.ap_create_id = activity['object']['id']
+                            post.ap_announce_id = activity['id']
+                            db.session.commit()
+
+                        activities_processed += 1
+                        if activities_processed >= 50:
+                            break
+                    community.post_count = activities_processed # todo: figure out why this value is not being saved
+                    db.session.commit()
+
+
 def community_url_exists(url) -> bool:
     community = Community.query.filter_by(ap_profile_id=url).first()
     return community is not None

app/main/routes.py

@@ -1,21 +1,27 @@
 from sqlalchemy.sql.operators import or_
 
 from app import db, cache
+from app.activitypub.util import default_context
 from app.constants import SUBSCRIPTION_PENDING, SUBSCRIPTION_MEMBER, POST_TYPE_IMAGE, POST_TYPE_LINK, SUBSCRIPTION_OWNER
 from app.main import bp
-from flask import g, session, flash, request, current_app, url_for, redirect, make_response
+from flask import g, session, flash, request, current_app, url_for, redirect, make_response, jsonify
 from flask_moment import moment
 from flask_login import current_user
 from flask_babel import _, get_locale
 from sqlalchemy import select, desc
 from sqlalchemy_searchable import search
-from app.utils import render_template, get_setting, gibberish, request_etag_matches, return_304, blocked_domains
+from app.utils import render_template, get_setting, gibberish, request_etag_matches, return_304, blocked_domains, \
+    ap_datetime
 from app.models import Community, CommunityMember, Post, Site, User
 
 
-@bp.route('/', methods=['GET', 'POST'])
+@bp.route('/', methods=['HEAD', 'GET', 'POST'])
 @bp.route('/index', methods=['GET', 'POST'])
 def index():
+    if 'application/ld+json' in request.headers.get('Accept', '') or 'application/activity+json' in request.headers.get(
+            'Accept', ''):
+        return activitypub_application()
+
     verification_warning()
 
     # If nothing has changed since their last visit, return HTTP 304
@@ -99,3 +105,20 @@ def test():
 def verification_warning():
     if hasattr(current_user, 'verified') and current_user.verified is False:
         flash(_('Please click the link in your email inbox to verify your account.'), 'warning')
+
+
+def activitypub_application():
+    application_data = {
+        '@context': default_context(),
+        'type': 'Application',
+        'id': f"https://{current_app.config['SERVER_NAME']}/",
+        'name': g.site.name,
+        'summary': g.site.description,
+        'published': ap_datetime(g.site.created_at),
+        'updated': ap_datetime(g.site.updated),
+        'inbox': f"https://{current_app.config['SERVER_NAME']}/site_inbox",
+        'outbox': f"https://{current_app.config['SERVER_NAME']}/site_outbox",
+    }
+    resp = jsonify(application_data)
+    resp.content_type = 'application/activity+json'
+    return resp

app/models.py

@@ -50,6 +50,11 @@ class File(db.Model):
             return ''
 
     def thumbnail_url(self):
+        if self.thumbnail_path is None:
+            if self.source_url:
+                return self.source_url
+            else:
+                return ''
         thumbnail_path = self.thumbnail_path[4:] if self.thumbnail_path.startswith('app/') else self.thumbnail_path
         return f"https://{current_app.config['SERVER_NAME']}/{thumbnail_path}"
 
@@ -68,8 +73,10 @@ class Community(db.Model):
     user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
     name = db.Column(db.String(256), index=True)
     title = db.Column(db.String(256))
-    description = db.Column(db.Text)
+    description = db.Column(db.Text)        # markdown
+    description_html = db.Column(db.Text)   # html equivalent of above markdown
     rules = db.Column(db.Text)
+    rules_html = db.Column(db.Text)
     content_warning = db.Column(db.Text)        # "Are you sure you want to view this community?"
     subscriptions_count = db.Column(db.Integer, default=0)
     post_count = db.Column(db.Integer, default=0)
@@ -173,6 +180,9 @@ class Community(db.Model):
     def is_moderator(self):
         return any(moderator.user_id == current_user.id for moderator in self.moderators())
 
+    def is_owner(self):
+        return any(moderator.user_id == current_user.id and moderator.is_owner for moderator in self.moderators())
+
     def profile_id(self):
         return self.ap_profile_id if self.ap_profile_id else f"https://{current_app.config['SERVER_NAME']}/c/{self.name}"
 
@@ -185,6 +195,17 @@ class Community(db.Model):
         else:
             return f"https://{current_app.config['SERVER_NAME']}/c/{self.ap_id}"
 
+    def delete_dependencies(self):
+        # this will be fine for remote communities but for local ones it is necessary to federate every deletion out to subscribers
+        for post in self.posts:
+            post.delete_dependencies()
+            db.session.delete(post)
+        db.session.query(CommunityBan).filter(CommunityBan.community_id == self.id).delete()
+        db.session.query(CommunityBlock).filter(CommunityBlock.community_id == self.id).delete()
+        db.session.query(CommunityJoinRequest).filter(CommunityJoinRequest.community_id == self.id).delete()
+        db.session.query(CommunityMember).filter(CommunityMember.community_id == self.id).delete()
+        db.session.query(Report).filter(Report.suspect_community_id == self.id).delete()
+
 
 user_role = db.Table('user_role',
     db.Column('user_id', db.Integer, db.ForeignKey('user.id')),
@@ -203,7 +224,8 @@ class User(UserMixin, db.Model):
     verification_token = db.Column(db.String(16), index=True)
     banned = db.Column(db.Boolean, default=False)
     deleted = db.Column(db.Boolean, default=False)
-    about = db.Column(db.Text)
+    about = db.Column(db.Text)      # markdown
+    about_html = db.Column(db.Text) # html
     keywords = db.Column(db.String(256))
     show_nsfw = db.Column(db.Boolean, default=False)
     show_nsfl = db.Column(db.Boolean, default=False)
@@ -225,6 +247,7 @@ class User(UserMixin, db.Model):
     bot = db.Column(db.Boolean, default=False)
     ignore_bots = db.Column(db.Boolean, default=False)
     unread_notifications = db.Column(db.Integer, default=0)
+    instance_id = db.Column(db.Integer, db.ForeignKey('instance.id'), index=True)
 
     avatar = db.relationship('File', lazy='joined', foreign_keys=[avatar_id], single_parent=True, cascade="all, delete-orphan")
     cover = db.relationship('File', lazy='joined', foreign_keys=[cover_id], single_parent=True, cascade="all, delete-orphan")
@@ -297,6 +320,12 @@ class User(UserMixin, db.Model):
     def is_local(self):
         return self.ap_id is None or self.ap_profile_id.startswith('https://' + current_app.config['SERVER_NAME'])
 
+    def is_admin(self):
+        for role in self.roles:
+            if role.name == 'Admin':
+                return True
+        return False
+
     def link(self) -> str:
         if self.is_local():
             return self.user_name

app/templates/community/community.html

@@ -99,7 +99,7 @@
                 </p>
             </div>
         </div>
-        {% if is_moderator %}
+        {% if is_moderator or current_user.is_admin() %}
         <div class="card mt-3">
             <div class="card-header">
                  <h2>{{ _('Community Settings') }}</h2>
@@ -107,6 +107,9 @@
             <div class="card-body">
                 <p><a href="#" class="btn btn-primary">{{ _('Moderate') }}</a></p>
                 <p><a href="#" class="btn btn-primary">{{ _('Settings') }}</a></p>
+                {% if community.is_owner() or current_user.is_admin() %}
+                    <p><a class="btn btn-primary btn-warning" href="{{ url_for('community.community_delete', community_id=community.id) }}" rel="nofollow">Delete community</a></p>
+                {% endif %}
             </div>
         </div>
         {% endif %}

app/templates/community/community_delete.html

@@ -0,0 +1,17 @@
+{% extends "base.html" %}
+{% from 'bootstrap/form.html' import render_form %}
+
+{% block app_content %}
+    <div class="row">
+        <div class="col col-login mx-auto">
+            <div class="card mt-5">
+                <div class="card-body p-6">
+                    <div class="card-title">{{ _('Delete "%(community_title)s"', community_title=community.title) }}</div>
+                    <div class="card-body">
+                        {{ render_form(form) }}
+                    </div>
+                </div>
+            </div>
+        </div>
+    </div>
+{% endblock %}

app/templates/post/_post_teaser.html

@@ -19,7 +19,8 @@
                     <span class="small">{{ render_username(post.author) }} · {{ moment(post.posted_at).fromNow() }}</span>
                 {% if post.image_id %}
                     <div class="thumbnail">
-                        <a href="{{ url_for('activitypub.post_ap', post_id=post.id) }}"><img src="{{ post.image.thumbnail_url() }}" alt="{{ post.image.alt_text }}"
+                        <a href="{{ url_for('activitypub.post_ap', post_id=post.id) }}"><img src="{{ post.image.thumbnail_url() }}"
+                                                                                             alt="{{ post.image.alt_text if post.image.alt_text else '' }}"
                                                                                              height="50" /></a>
                     </div>
                 {% endif %}

interests.txt

@@ -165,3 +165,9 @@ mental health
     https://beehaw.org/c/neurodivergence
     https://lemmy.blahaj.zone/c/bipolar
     
+health
+    https://lemmy.ml/c/coronavirus
+    https://mander.xyz/c/medicine
+    https://lemmy.world/c/health
+    https://lemmy.ml/c/health
+    https://mander.xyz/c/medicine

migrations/versions/c12823f18553_html_versions.py

@@ -0,0 +1,46 @@
+"""html versions
+
+Revision ID: c12823f18553
+Revises: 72f3326bdf54
+Create Date: 2023-12-21 20:21:55.039590
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'c12823f18553'
+down_revision = '72f3326bdf54'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('community', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('description_html', sa.Text(), nullable=True))
+        batch_op.add_column(sa.Column('rules_html', sa.Text(), nullable=True))
+
+    with op.batch_alter_table('user', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('about_html', sa.Text(), nullable=True))
+        batch_op.add_column(sa.Column('instance_id', sa.Integer(), nullable=True))
+        batch_op.create_index(batch_op.f('ix_user_instance_id'), ['instance_id'], unique=False)
+        batch_op.create_foreign_key(None, 'instance', ['instance_id'], ['id'])
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('user', schema=None) as batch_op:
+        batch_op.drop_constraint(None, type_='foreignkey')
+        batch_op.drop_index(batch_op.f('ix_user_instance_id'))
+        batch_op.drop_column('instance_id')
+        batch_op.drop_column('about_html')
+
+    with op.batch_alter_table('community', schema=None) as batch_op:
+        batch_op.drop_column('rules_html')
+        batch_op.drop_column('description_html')
+
+    # ### end Alembic commands ###

pyfedi.py

@@ -14,7 +14,7 @@ cli.register(app)
 
 
 @app.context_processor
-def app_context_processor():  # NB there needs to be an identical function in cb.wsgi to make this work in production
+def app_context_processor():
     def getmtime(filename):
         return os.path.getmtime('app/static/' + filename)
     return dict(getmtime=getmtime, post_type_link=POST_TYPE_LINK, post_type_image=POST_TYPE_IMAGE, post_type_article=POST_TYPE_ARTICLE)