Merge branch 'main' of https://codeberg.org/saint/pyfedi

INSTALL.md

@@ -300,4 +300,44 @@ Inspect log files at:
 
     /var/log/celery/*
     /var/log/nginx/*
-    /your_piefed_installation/logs/pyfedi.log
+    /your_piefed_installation/logs/pyfedi.log
+
+
+### Nginx
+
+You need a reverse proxy that sends all traffic to port 5000. Something like:
+
+    upstream app_server {
+        # fail_timeout=0 means we always retry an upstream even if it failed
+        # to return a good HTTP response
+
+        # for UNIX domain socket setups
+        # server unix:/tmp/gunicorn.sock fail_timeout=0;
+
+        # for a TCP configuration
+        server 127.0.0.1:5000 fail_timeout=0;
+        keepalive 4;
+    }
+
+    server {
+        server_name piefed.social
+        root /whatever
+
+        keepalive_timeout 5;
+        ssi off;
+
+        location / {
+            # Proxy all requests to Gunicorn
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header Host $http_host;
+            proxy_redirect off;
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
+            proxy_pass http://app_server;
+            ssi off;
+        }
+    }
+
+The above is not a complete configuration - you will want to add more settings for SSL, etc.
+

app/activitypub/util.py

@@ -234,7 +234,7 @@ def find_actor_or_create(actor: str, create_if_not_found=True) -> Union[User, Co
                 if isinstance(user, User):
                     refresh_user_profile(user.id)
                 elif isinstance(user, Community):
-                    # todo: refresh community profile also, not just instance_profile
+                    refresh_community_profile(user.id)
                     refresh_instance_profile(user.instance_id)
         return user
     else:   # User does not exist in the DB, it's going to need to be created from it's remote home instance
@@ -357,6 +357,112 @@ def refresh_user_profile_task(user_id):
                 make_image_sizes(user.cover_id, 700, 1600, 'users')
 
 
+def refresh_community_profile(community_id):
+    if current_app.debug:
+        refresh_community_profile_task(community_id)
+    else:
+        refresh_community_profile_task.apply_async(args=(community_id,), countdown=randint(1, 10))
+
+
+@celery.task
+def refresh_community_profile_task(community_id):
+    community = Community.query.get(community_id)
+    if community and not community.is_local():
+        try:
+            actor_data = get_request(community.ap_profile_id, headers={'Accept': 'application/activity+json'})
+        except requests.exceptions.ReadTimeout:
+            time.sleep(randint(3, 10))
+            actor_data = get_request(community.ap_profile_id, headers={'Accept': 'application/activity+json'})
+        if actor_data.status_code == 200:
+            activity_json = actor_data.json()
+            actor_data.close()
+
+            if 'attributedTo' in activity_json:  # lemmy and mbin
+                mods_url = activity_json['attributedTo']
+            elif 'moderators' in activity_json:  # kbin
+                mods_url = activity_json['moderators']
+            else:
+                mods_url = None
+
+            community.nsfw = activity_json['sensitive']
+            if 'nsfl' in activity_json and activity_json['nsfl']:
+                community.nsfl = activity_json['nsfl']
+            community.title = activity_json['name']
+            community.description = activity_json['summary'] if 'summary' in activity_json else ''
+            community.rules = activity_json['rules'] if 'rules' in activity_json else ''
+            community.rules_html = markdown_to_html(activity_json['rules'] if 'rules' in activity_json else '')
+            community.restricted_to_mods = activity_json['postingRestrictedToMods']
+            community.new_mods_wanted = activity_json['newModsWanted'] if 'newModsWanted' in activity_json else False
+            community.private_mods = activity_json['privateMods'] if 'privateMods' in activity_json else False
+            community.ap_moderators_url = mods_url
+            community.ap_fetched_at = utcnow()
+            community.public_key=activity_json['publicKey']['publicKeyPem']
+
+            if 'source' in activity_json and \
+                    activity_json['source']['mediaType'] == 'text/markdown':
+                community.description = activity_json['source']['content']
+                community.description_html = markdown_to_html(community.description)
+            elif 'content' in activity_json:
+                community.description_html = allowlist_html(activity_json['content'])
+                community.description = html_to_markdown(community.description_html)
+
+            icon_changed = cover_changed = False
+            if 'icon' in activity_json:
+                if community.icon_id and activity_json['icon']['url'] != community.icon.source_url:
+                    community.icon.delete_from_disk()
+                icon = File(source_url=activity_json['icon']['url'])
+                community.icon = icon
+                db.session.add(icon)
+                icon_changed = True
+            if 'image' in activity_json:
+                if community.image_id and activity_json['image']['url'] != community.image.source_url:
+                    community.image.delete_from_disk()
+                image = File(source_url=activity_json['image']['url'])
+                community.image = image
+                db.session.add(image)
+                cover_changed = True
+            db.session.commit()
+            if community.icon_id and icon_changed:
+                make_image_sizes(community.icon_id, 60, 250, 'communities')
+            if community.image_id and cover_changed:
+                make_image_sizes(community.image_id, 700, 1600, 'communities')
+
+            if community.ap_moderators_url:
+                mods_request = get_request(community.ap_moderators_url, headers={'Accept': 'application/activity+json'})
+                if mods_request.status_code == 200:
+                    mods_data = mods_request.json()
+                    mods_request.close()
+                    if mods_data and mods_data['type'] == 'OrderedCollection' and 'orderedItems' in mods_data:
+                        for actor in mods_data['orderedItems']:
+                            time.sleep(0.5)
+                            user = find_actor_or_create(actor)
+                            if user:
+                                existing_membership = CommunityMember.query.filter_by(community_id=community.id,
+                                                                                      user_id=user.id).first()
+                                if existing_membership:
+                                    existing_membership.is_moderator = True
+                                    db.session.commit()
+                                else:
+                                    new_membership = CommunityMember(community_id=community.id, user_id=user.id,
+                                                                     is_moderator=True)
+                                    db.session.add(new_membership)
+                                    db.session.commit()
+
+                        # Remove people who are no longer mods
+                        for member in CommunityMember.query.filter_by(community_id=community.id, is_moderator=True).all():
+                            member_user = User.query.get(member.user_id)
+                            is_mod = False
+                            for actor in mods_data['orderedItems']:
+                                if actor.lower() == member_user.profile_id().lower():
+                                    is_mod = True
+                                    break
+                            if not is_mod:
+                                db.session.query(CommunityMember).filter_by(community_id=community.id,
+                                                                            user_id=member_user.id,
+                                                                            is_moderator=True).delete()
+                                db.session.commit()
+
+
 def actor_json_to_model(activity_json, address, server):
     if activity_json['type'] == 'Person':
         user = User(user_name=activity_json['preferredUsername'],

app/admin/routes.py

@@ -445,6 +445,36 @@ def admin_users():
                            )
 
 
+@bp.route('/users/trash', methods=['GET'])
+@login_required
+@permission_required('administer all users')
+def admin_users_trash():
+
+    page = request.args.get('page', 1, type=int)
+    search = request.args.get('search', '')
+    local_remote = request.args.get('local_remote', '')
+
+    users = User.query.filter_by(deleted=False)
+    if local_remote == 'local':
+        users = users.filter_by(ap_id=None)
+    if local_remote == 'remote':
+        users = users.filter(User.ap_id != None)
+    if search:
+        users = users.filter(User.email.ilike(f"%{search}%"))
+    users = users.filter(User.reputation < -10)
+    users = users.order_by(User.reputation).paginate(page=page, per_page=1000, error_out=False)
+
+    next_url = url_for('admin.admin_users_trash', page=users.next_num) if users.has_next else None
+    prev_url = url_for('admin.admin_users_trash', page=users.prev_num) if users.has_prev and page != 1 else None
+
+    return render_template('admin/users.html', title=_('Problematic users'), next_url=next_url, prev_url=prev_url, users=users,
+                           local_remote=local_remote, search=search,
+                           moderating_communities=moderating_communities(current_user.get_id()),
+                           joined_communities=joined_communities(current_user.get_id()),
+                           site=g.site
+                           )
+
+
 @bp.route('/approve_registrations', methods=['GET'])
 @login_required
 @permission_required('approve registrations')

app/cli.py

@@ -142,8 +142,7 @@ def register(app):
     def daily_maintenance():
         with app.app_context():
             """Remove activity older than 3 days"""
-            db.session.query(ActivityPubLog).filter(
-                ActivityPubLog.created_at < utcnow() - timedelta(days=3)).delete()
+            db.session.query(ActivityPubLog).filter(ActivityPubLog.created_at < utcnow() - timedelta(days=3)).delete()
             db.session.commit()
 
     @app.cli.command("spaceusage")

app/main/routes.py

@@ -4,10 +4,12 @@ from math import log
 from random import randint
 
 import flask
+import markdown2
 from sqlalchemy.sql.operators import or_, and_
 
 from app import db, cache
-from app.activitypub.util import default_context, make_image_sizes_async, refresh_user_profile, find_actor_or_create
+from app.activitypub.util import default_context, make_image_sizes_async, refresh_user_profile, find_actor_or_create, \
+    refresh_community_profile_task
 from app.constants import SUBSCRIPTION_PENDING, SUBSCRIPTION_MEMBER, POST_TYPE_IMAGE, POST_TYPE_LINK, \
     SUBSCRIPTION_OWNER, SUBSCRIPTION_MODERATOR
 from app.email import send_email, send_welcome_email
@@ -21,7 +23,7 @@ from sqlalchemy import select, desc, text
 from sqlalchemy_searchable import search
 from app.utils import render_template, get_setting, gibberish, request_etag_matches, return_304, blocked_domains, \
     ap_datetime, ip_address, retrieve_block_list, shorten_string, markdown_to_text, user_filters_home, \
-    joined_communities, moderating_communities, parse_page, theme_list, get_request
+    joined_communities, moderating_communities, parse_page, theme_list, get_request, markdown_to_html, allowlist_html
 from app.models import Community, CommunityMember, Post, Site, User, utcnow, Domain, Topic, File, Instance, \
     InstanceRole, Notification
 from PIL import Image
@@ -132,8 +134,8 @@ def home_page(type, sort):
                            low_bandwidth=low_bandwidth,
                            SUBSCRIPTION_PENDING=SUBSCRIPTION_PENDING, SUBSCRIPTION_MEMBER=SUBSCRIPTION_MEMBER,
                            etag=f"{type}_{sort}_{hash(str(g.site.last_active))}", next_url=next_url, prev_url=prev_url,
-                           rss_feed=f"https://{current_app.config['SERVER_NAME']}/feed",
-                           rss_feed_name=f"Posts on " + g.site.name,
+                           #rss_feed=f"https://{current_app.config['SERVER_NAME']}/feed",
+                           #rss_feed_name=f"Posts on " + g.site.name,
                            title=f"{g.site.name} - {g.site.description}",
                            description=shorten_string(markdown_to_text(g.site.sidebar), 150),
                            content_filters=content_filters, type=type, sort=sort,
@@ -259,10 +261,9 @@ def list_files(directory):
 
 @bp.route('/test')
 def test():
-    u = User.query.get(1)
-    send_welcome_email(u, False)
+    x = find_actor_or_create('artporn@lemm.ee')
+    return 'ok'
 
-    return ''
     users_to_notify = User.query.join(Notification, User.id == Notification.user_id).filter(
             User.ap_id == None,
             Notification.created_at > User.last_seen,

app/post/routes.py

@@ -176,7 +176,7 @@ def show_post(post_id: int):
                            canonical=post.ap_id, form=form, replies=replies, THREAD_CUTOFF_DEPTH=constants.THREAD_CUTOFF_DEPTH,
                            description=description, og_image=og_image, POST_TYPE_IMAGE=constants.POST_TYPE_IMAGE,
                            POST_TYPE_LINK=constants.POST_TYPE_LINK, POST_TYPE_ARTICLE=constants.POST_TYPE_ARTICLE,
-                           etag=f"{post.id}{sort}_{hash(post.last_active)}", markdown_editor=current_user.markdown_editor,
+                           etag=f"{post.id}{sort}_{hash(post.last_active)}", markdown_editor=current_user.is_authenticated and current_user.markdown_editor,
                            low_bandwidth=request.cookies.get('low_bandwidth', '0') == '1', SUBSCRIPTION_MEMBER=SUBSCRIPTION_MEMBER,
                            moderating_communities=moderating_communities(current_user.get_id()),
                            joined_communities=joined_communities(current_user.get_id()),
@@ -368,7 +368,7 @@ def continue_discussion(post_id, comment_id):
     replies = get_comment_branch(post.id, comment.id, 'top')
 
     return render_template('post/continue_discussion.html', title=_('Discussing %(title)s', title=post.title), post=post,
-                           is_moderator=is_moderator, comment=comment, replies=replies, markdown_editor=current_user.markdown_editor,
+                           is_moderator=is_moderator, comment=comment, replies=replies, markdown_editor=current_user.is_authenticated and current_user.markdown_editor,
                            moderating_communities=moderating_communities(current_user.get_id()),
                            joined_communities=joined_communities(current_user.get_id()), community=post.community,
                            inoculation=inoculation[randint(0, len(inoculation) - 1)])
@@ -531,7 +531,7 @@ def add_reply(post_id: int, comment_id: int):
     else:
         form.notify_author.data = True
         return render_template('post/add_reply.html', title=_('Discussing %(title)s', title=post.title), post=post,
-                               is_moderator=is_moderator, form=form, comment=in_reply_to, markdown_editor=current_user.markdown_editor,
+                               is_moderator=is_moderator, form=form, comment=in_reply_to, markdown_editor=current_user.is_authenticated and current_user.markdown_editor,
                                moderating_communities=moderating_communities(current_user.get_id()),
                                joined_communities = joined_communities(current_user.id),
                                inoculation=inoculation[randint(0, len(inoculation) - 1)])

app/templates/admin/_nav.html

@@ -5,6 +5,7 @@
     <a href="{{ url_for('admin.admin_communities') }}">{{ _('Communities') }}</a> |
     <a href="{{ url_for('admin.admin_topics') }}">{{ _('Topics') }}</a> |
     <a href="{{ url_for('admin.admin_users', local_remote='local') }}">{{ _('Users') }}</a> |
+    <a href="{{ url_for('admin.admin_users_trash', local_remote='local') }}">{{ _('Watch') }}</a> |
     {% if site.registration_mode == 'RequireApplication' %}
         <a href="{{ url_for('admin.admin_approve_registrations') }}">{{ _('Registration applications') }}</a> |
     {% endif %}

app/templates/index.html

@@ -67,9 +67,11 @@
             <div class="card-body">
                 <p><strong>{{ g.site.description|safe }}</strong></p>
                 <p>{{ g.site.sidebar|safe }}</p>
+                {% if rss_feed %}
                 <p class="mt-4">
                     <a class="no-underline" href="{{ rss_feed }}" rel="nofollow"><span class="fe fe-rss"></span> RSS feed</a>
                 </p>
+                {% endif %}
             </div>
         </div>
 

app/utils.py

@@ -162,7 +162,7 @@ def allowlist_html(html: str) -> str:
     if html is None or html == '':
         return ''
     allowed_tags = ['p', 'strong', 'a', 'ul', 'ol', 'li', 'em', 'blockquote', 'cite', 'br', 'h3', 'h4', 'h5', 'pre',
-                    'code', 'img', 'details', 'summary']
+                    'code', 'img', 'details', 'summary', 'table', 'tr', 'td', 'th', 'tbody', 'thead']
     # Parse the HTML using BeautifulSoup
     soup = BeautifulSoup(html, 'html.parser')
 
@@ -201,6 +201,8 @@ def allowlist_html(html: str) -> str:
             # Add loading=lazy to images
             if tag.name == 'img':
                 tag.attrs['loading'] = 'lazy'
+            if tag.name == 'table':
+                tag.attrs['class'] = 'table'
 
     return str(soup)
 
@@ -242,7 +244,7 @@ def html_to_markdown_worker(element, indent_level=0):
 
 def markdown_to_html(markdown_text) -> str:
     if markdown_text:
-        return allowlist_html(markdown2.markdown(markdown_text, safe_mode=True, extras={'middle-word-em': False}))
+        return allowlist_html(markdown2.markdown(markdown_text, safe_mode=True, extras={'middle-word-em': False, 'tables': True}))
     else:
         return ''
 

config.py

@@ -34,7 +34,7 @@ class Config(object):
     SQLALCHEMY_ECHO = False     # set to true to see SQL in console
     WTF_CSRF_TIME_LIMIT = None  # a value of None ensures csrf token is valid for the lifetime of the session
 
-    BOUNCE_HOST = os.environ.get('BOUNCE_HOST')
-    BOUNCE_USERNAME = os.environ.get('BOUNCE_USERNAME')
-    BOUNCE_PASSWORD = os.environ.get('BOUNCE_PASSWORD')
+    BOUNCE_HOST = os.environ.get('BOUNCE_HOST') or ''
+    BOUNCE_USERNAME = os.environ.get('BOUNCE_USERNAME') or ''
+    BOUNCE_PASSWORD = os.environ.get('BOUNCE_PASSWORD') or ''
 

env.sample

@@ -10,3 +10,7 @@ CACHE_TYPE='FileSystemCache'
 CACHE_DIR='/dev/shm/pyfedi'
 CELERY_BROKER_URL='redis://localhost:6379/1'
 CACHE_REDIS_URL='redis://localhost:6379/1'
+
+BOUNCE_HOST=''
+BOUNCE_USERNAME=''
+BOUNCE_PASSWORD=''