Merge remote-tracking branch 'upstream/main'

app/activitypub/routes.py

@@ -203,7 +203,7 @@ def user_profile(actor):
                             "outbox": f"https://{server}/u/{actor}/outbox",
                             "discoverable": user.searchable,
                             "indexable": user.indexable,
-                            "manuallyApprovesFollowers": user.ap_manually_approves_followers,
+                            "manuallyApprovesFollowers": False if not user.ap_manually_approves_followers else user.ap_manually_approves_followers,
                             "publicKey": {
                                 "id": f"https://{server}/u/{actor}#main-key",
                                 "owner": f"https://{server}/u/{actor}",
@@ -666,24 +666,48 @@ def process_inbox_request(request_json, activitypublog_id, ip_address):
                             target_ap_id = request_json['object']['object']['object']           # object object object!
                             post = undo_vote(activity_log, comment, post, target_ap_id, user)
                             activity_log.result = 'success'
-                    elif request_json['object']['type'] == 'Add':
+                    elif request_json['object']['type'] == 'Add' and 'target' in request_json['object']:
                         activity_log.activity_type = request_json['object']['type']
-                        featured_url = Community.query.filter(Community.ap_public_url == request_json['actor']).first().ap_featured_url
+                        target = request_json['object']['target']
-                        if featured_url:
+                        community = Community.query.filter_by(ap_public_url=request_json['actor']).first()
-                           if 'target' in request_json['object'] and featured_url == request_json['object']['target']:
+                        if community:
-                                post = Post.query.filter(Post.ap_id == request_json['object']['object']).first()
+                            featured_url = community.ap_featured_url
+                            moderators_url = community.ap_moderators_url
+                            if target == featured_url:
+                                post = Post.query.filter_by(ap_id=request_json['object']['object']).first()
                                 if post:
                                     post.sticky = True
                                     activity_log.result = 'success'
-                    elif request_json['object']['type'] == 'Remove':
+                            if target == moderators_url:
+                                user = find_actor_or_create(request_json['object']['object'])
+                                if user:
+                                    existing_membership = CommunityMember.query.filter_by(community_id=community.id, user_id=user.id).first()
+                                    if existing_membership:
+                                        existing_membership.is_moderator = True
+                                    else:
+                                        new_membership = CommunityMember(community_id=community.id, user_id=user.id, is_moderator=True)
+                                        db.session.add(new_membership)
+                                        db.session.commit()
+                                    activity_log.result = 'success'
+                    elif request_json['object']['type'] == 'Remove' and 'target' in request_json['object']:
                         activity_log.activity_type = request_json['object']['type']
-                        featured_url = Community.query.filter(Community.ap_public_url == request_json['actor']).first().ap_featured_url
+                        target = request_json['object']['target']
-                        if featured_url:
+                        community = Community.query.filter_by(ap_public_url=request_json['actor']).first()
-                           if 'target' in request_json['object'] and featured_url == request_json['object']['target']:
+                        if community:
-                                post = Post.query.filter(Post.ap_id == request_json['object']['object']).first()
+                            featured_url = community.ap_featured_url
+                            moderators_url = community.ap_moderators_url
+                            if target == featured_url:
+                                post = Post.query.filter_by(ap_id=request_json['object']['object']).first()
                                 if post:
                                     post.sticky = False
                                     activity_log.result = 'success'
+                            if target == moderators_url:
+                                user = find_actor_or_create(request_json['object']['object'], create_if_not_found=False)
+                                if user:
+                                    existing_membership = CommunityMember.query.filter_by(community_id=community.id, user_id=user.id).first()
+                                    if existing_membership:
+                                        existing_membership.is_moderator = False
+                                    activity_log.result = 'success'
                     else:
                         activity_log.exception_message = 'Invalid type for Announce'
 

app/activitypub/util.py

@@ -24,7 +24,8 @@ import pytesseract
 
 from app.utils import get_request, allowlist_html, html_to_markdown, get_setting, ap_datetime, markdown_to_html, \
     is_image_url, domain_from_url, gibberish, ensure_directory_exists, markdown_to_text, head_request, post_ranking, \
-    shorten_string, reply_already_exists, reply_is_just_link_to_gif_reaction, confidence, remove_tracking_from_link
+    shorten_string, reply_already_exists, reply_is_just_link_to_gif_reaction, confidence, remove_tracking_from_link, \
+    blocked_phrases
 
 
 def public_key():
@@ -1177,6 +1178,11 @@ def create_post_reply(activity_log: ActivityPubLog, community: Community, in_rep
             post_reply.body_html = allowlist_html(request_json['object']['content'])
             post_reply.body = html_to_markdown(post_reply.body_html)
         if post_id is not None:
+            # Discard post_reply if it contains certain phrases. Good for stopping spam floods.
+            if post_reply.body:
+                for blocked_phrase in blocked_phrases():
+                    if blocked_phrase in post_reply.body:
+                        return None
             post = Post.query.get(post_id)
             if post.comments_enabled:
                 anchor = None
@@ -1270,6 +1276,15 @@ def create_post(activity_log: ActivityPubLog, community: Community, request_json
     elif 'content' in request_json['object'] and request_json['object']['content'] is not None: # Kbin
         post.body_html = allowlist_html(request_json['object']['content'])
         post.body = html_to_markdown(post.body_html)
+    # Discard post if it contains certain phrases. Good for stopping spam floods.
+    blocked_phrases_list = blocked_phrases()
+    for blocked_phrase in blocked_phrases_list:
+        if blocked_phrase in post.title:
+            return None
+    if post.body:
+        for blocked_phrase in blocked_phrases_list:
+            if blocked_phrase in post.body:
+                return None
     if 'attachment' in request_json['object'] and len(request_json['object']['attachment']) > 0 and \
             'type' in request_json['object']['attachment'][0]:
         if request_json['object']['attachment'][0]['type'] == 'Link':

app/admin/forms.py

@@ -31,6 +31,7 @@ class SiteMiscForm(FlaskForm):
     types = [('Open', _l('Open')), ('RequireApplication', _l('Require application')), ('Closed', _l('Closed'))]
     registration_mode = SelectField(_l('Registration mode'), choices=types, default=1, coerce=str)
     application_question = TextAreaField(_l('Question to ask people applying for an account'))
+    auto_decline_referrers = TextAreaField(_l('Block registrations from these referrers (one per line)'))
     log_activitypub_json = BooleanField(_l('Log ActivityPub JSON for debugging'))
     default_theme = SelectField(_l('Default theme'), coerce=str)
     submit = SubmitField(_l('Save'))
@@ -41,6 +42,7 @@ class FederationForm(FlaskForm):
     allowlist = TextAreaField(_l('Allow federation with these instances'))
     use_blocklist = BooleanField(_l('Blocklist instead of allowlist'))
     blocklist = TextAreaField(_l('Deny federation with these instances'))
+    blocked_phrases = TextAreaField(_l('Discard all posts and comments with these phrases (one per line)'))
     submit = SubmitField(_l('Save'))
 
 

app/admin/routes.py

@@ -6,10 +6,10 @@ from flask_login import login_required, current_user
 from flask_babel import _
 from sqlalchemy import text, desc
 
-from app import db, celery
+from app import db, celery, cache
 from app.activitypub.routes import process_inbox_request, process_delete_request
 from app.activitypub.signature import post_request
-from app.activitypub.util import default_context
+from app.activitypub.util import default_context, instance_allowed, instance_blocked
 from app.admin.forms import FederationForm, SiteMiscForm, SiteProfileForm, EditCommunityForm, EditUserForm, \
     EditTopicForm, SendNewsletterForm, AddUserForm
 from app.admin.util import unsubscribe_from_everything_then_delete, unsubscribe_from_community, send_newsletter, \
@@ -18,7 +18,7 @@ from app.community.util import save_icon_file, save_banner_file
 from app.models import AllowedInstances, BannedInstances, ActivityPubLog, utcnow, Site, Community, CommunityMember, \
     User, Instance, File, Report, Topic, UserRegistration, Role, Post
 from app.utils import render_template, permission_required, set_setting, get_setting, gibberish, markdown_to_html, \
-    moderating_communities, joined_communities, finalize_user_setup, theme_list
+    moderating_communities, joined_communities, finalize_user_setup, theme_list, blocked_phrases, blocked_referrers
 from app.admin import bp
 
 
@@ -80,12 +80,14 @@ def admin_misc():
         site.reports_email_admins = form.reports_email_admins.data
         site.registration_mode = form.registration_mode.data
         site.application_question = form.application_question.data
+        site.auto_decline_referrers = form.auto_decline_referrers.data
         site.log_activitypub_json = form.log_activitypub_json.data
         site.updated = utcnow()
         site.default_theme = form.default_theme.data
         if site.id is None:
             db.session.add(site)
         db.session.commit()
+        cache.delete_memoized(blocked_referrers)
         flash('Settings saved.')
     elif request.method == 'GET':
         form.enable_downvotes.data = site.enable_downvotes
@@ -97,6 +99,7 @@ def admin_misc():
         form.reports_email_admins.data = site.reports_email_admins
         form.registration_mode.data = site.registration_mode
         form.application_question.data = site.application_question
+        form.auto_decline_referrers.data = site.auto_decline_referrers
         form.log_activitypub_json.data = site.log_activitypub_json
         form.default_theme.data = site.default_theme if site.default_theme is not None else ''
     return render_template('admin/misc.html', title=_('Misc settings'), form=form,
@@ -123,13 +126,18 @@ def admin_federation():
             for allow in form.allowlist.data.split('\n'):
                 if allow.strip():
                     db.session.add(AllowedInstances(domain=allow.strip()))
+                    cache.delete_memoized(instance_allowed, allow.strip())
         if form.use_blocklist.data:
             set_setting('use_allowlist', False)
             db.session.execute(text('DELETE FROM banned_instances'))
             for banned in form.blocklist.data.split('\n'):
                 if banned.strip():
                     db.session.add(BannedInstances(domain=banned.strip()))
+                    cache.delete_memoized(instance_blocked, banned.strip())
+        site.blocked_phrases = form.blocked_phrases.data
+        cache.delete_memoized(blocked_phrases)
         db.session.commit()
+
         flash(_('Admin settings saved'))
 
     elif request.method == 'GET':
@@ -139,6 +147,7 @@ def admin_federation():
         form.blocklist.data = '\n'.join([instance.domain for instance in instances])
         instances = AllowedInstances.query.all()
         form.allowlist.data = '\n'.join([instance.domain for instance in instances])
+        form.blocked_phrases.data = site.blocked_phrases
 
     return render_template('admin/federation.html', title=_('Federation settings'), form=form,
                            moderating_communities=moderating_communities(current_user.get_id()),

app/auth/routes.py

@@ -12,7 +12,7 @@ from app.auth.util import random_token, normalize_utf
 from app.email import send_verification_email, send_password_reset_email
 from app.models import User, utcnow, IpBan, UserRegistration, Notification, Site
 from app.utils import render_template, ip_address, user_ip_banned, user_cookie_banned, banned_ip_addresses, \
-    finalize_user_setup
+    finalize_user_setup, blocked_referrers
 
 
 @bp.route('/login', methods=['GET', 'POST'])
@@ -98,6 +98,11 @@ def register():
             if form.user_name.data in disallowed_usernames:
                 flash(_('Sorry, you cannot use that user name'), 'error')
             else:
+                for referrer in blocked_referrers():
+                    if referrer in session.get('Referer'):
+                        resp = make_response(redirect(url_for('auth.please_wait')))
+                        resp.set_cookie('sesion', '17489047567495', expires=datetime(year=2099, month=12, day=30))
+                        return resp
                 verification_token = random_token(16)
                 form.user_name.data = form.user_name.data.strip()
                 before_normalize = form.user_name.data

app/community/util.py

@@ -16,7 +16,7 @@ from app.models import Community, File, BannedInstances, PostReply, PostVote, Po
     Instance, Notification, User, ActivityPubLog
 from app.utils import get_request, gibberish, markdown_to_html, domain_from_url, allowlist_html, \
     html_to_markdown, is_image_url, ensure_directory_exists, inbox_domain, post_ranking, shorten_string, parse_page, \
-    remove_tracking_from_link, ap_datetime, instance_banned
+    remove_tracking_from_link, ap_datetime, instance_banned, blocked_phrases
 from sqlalchemy import func, desc
 import os
 
@@ -122,7 +122,7 @@ def retrieve_mods_and_backfill(community_id: int):
                         c.last_active = Post.query.filter(Post.community_id == community_id).order_by(desc(Post.posted_at)).first().posted_at
                     db.session.commit()
                 if community.ap_featured_url:
-                    featured_request = get_request(community.ap_featured_url, headers={'Accept': 'application/activityjson'})
+                    featured_request = get_request(community.ap_featured_url, headers={'Accept': 'application/activity+json'})
                     if featured_request.status_code == 200:
                         featured_data = featured_request.json()
                         featured_request.close()
@@ -299,6 +299,19 @@ def save_post(form, post: Post):
         if current_user.reputation < -100:
             post.score = -1
         post.ranking = post_ranking(post.score, utcnow())
+
+        # Filter by phrase
+        blocked_phrases_list = blocked_phrases()
+        for blocked_phrase in blocked_phrases_list:
+            if blocked_phrase in post.title:
+                abort(401)
+                return
+        if post.body:
+            for blocked_phrase in blocked_phrases_list:
+                if blocked_phrase in post.body:
+                    abort(401)
+                    return
+
         db.session.add(post)
 
     g.site.last_active = utcnow()

app/models.py

@@ -488,7 +488,7 @@ class User(UserMixin, db.Model):
     ap_fetched_at = db.Column(db.DateTime)
     ap_followers_url = db.Column(db.String(255))
     ap_preferred_username = db.Column(db.String(255))
-    ap_manually_approves_followers = db.Column(db.Boolean)
+    ap_manually_approves_followers = db.Column(db.Boolean, default=False)
     ap_deleted_at = db.Column(db.DateTime)
     ap_inbox_url = db.Column(db.String(255))
     ap_domain = db.Column(db.String(255))
@@ -1170,7 +1170,8 @@ class Site(db.Model):
     allow_or_block_list = db.Column(db.Integer, default=2)  # 1 = allow list, 2 = block list
     allowlist = db.Column(db.Text, default='')
     blocklist = db.Column(db.Text, default='')
-    auto_decline_referrers = db.Column(db.Text, default='rdrama.net')
+    blocked_phrases = db.Column(db.Text, default='')                     # discard incoming content with these phrases
+    auto_decline_referrers = db.Column(db.Text, default='rdrama.net\nahrefs.com')   # automatically decline registration requests if the referrer is one of these
     created_at = db.Column(db.DateTime, default=utcnow)
     updated = db.Column(db.DateTime, default=utcnow)
     last_active = db.Column(db.DateTime, default=utcnow)

app/post/routes.py

@@ -18,13 +18,13 @@ from app.post.util import post_replies, get_comment_branch, post_reply_count
 from app.constants import SUBSCRIPTION_MEMBER, POST_TYPE_LINK, POST_TYPE_IMAGE
 from app.models import Post, PostReply, \
     PostReplyVote, PostVote, Notification, utcnow, UserBlock, DomainBlock, InstanceBlock, Report, Site, Community, \
-    Topic
+    Topic, User
 from app.post import bp
 from app.utils import get_setting, render_template, allowlist_html, markdown_to_html, validation_required, \
     shorten_string, markdown_to_text, gibberish, ap_datetime, return_304, \
     request_etag_matches, ip_address, user_ip_banned, instance_banned, can_downvote, can_upvote, post_ranking, \
     reply_already_exists, reply_is_just_link_to_gif_reaction, confidence, moderating_communities, joined_communities, \
-    blocked_instances, blocked_domains, community_moderators
+    blocked_instances, blocked_domains, community_moderators, blocked_phrases
 
 
 def show_post(post_id: int):
@@ -47,6 +47,12 @@ def show_post(post_id: int):
     mods = community_moderators(community.id)
     is_moderator = current_user.is_authenticated and any(mod.user_id == current_user.id for mod in mods)
 
+    if community.private_mods:
+        mod_list = []
+    else:
+        mod_user_ids = [mod.user_id for mod in mods]
+        mod_list = User.query.filter(User.id.in_(mod_user_ids)).all()
+
     # handle top-level comments/replies
     form = NewReplyForm()
     if current_user.is_authenticated and current_user.verified and form.validate_on_submit():
@@ -213,7 +219,7 @@ def show_post(post_id: int):
         breadcrumbs.append(breadcrumb)
 
     response = render_template('post/post.html', title=post.title, post=post, is_moderator=is_moderator, community=post.community,
-                           breadcrumbs=breadcrumbs, related_communities=related_communities,
+                           breadcrumbs=breadcrumbs, related_communities=related_communities, mods=mod_list,
                            canonical=post.ap_id, form=form, replies=replies, THREAD_CUTOFF_DEPTH=constants.THREAD_CUTOFF_DEPTH,
                            description=description, og_image=og_image, POST_TYPE_IMAGE=constants.POST_TYPE_IMAGE,
                            POST_TYPE_LINK=constants.POST_TYPE_LINK, POST_TYPE_ARTICLE=constants.POST_TYPE_ARTICLE,
@@ -409,9 +415,14 @@ def continue_discussion(post_id, comment_id):
         abort(404)
     mods = post.community.moderators()
     is_moderator = current_user.is_authenticated and any(mod.user_id == current_user.id for mod in mods)
+    if post.community.private_mods:
+        mod_list = []
+    else:
+        mod_user_ids = [mod.user_id for mod in mods]
+        mod_list = User.query.filter(User.id.in_(mod_user_ids)).all()
     replies = get_comment_branch(post.id, comment.id, 'top')
 
-    response = render_template('post/continue_discussion.html', title=_('Discussing %(title)s', title=post.title), post=post,
+    response = render_template('post/continue_discussion.html', title=_('Discussing %(title)s', title=post.title), post=post, mods=mod_list,
                            is_moderator=is_moderator, comment=comment, replies=replies, markdown_editor=current_user.is_authenticated and current_user.markdown_editor,
                            moderating_communities=moderating_communities(current_user.get_id()),
                            joined_communities=joined_communities(current_user.get_id()), community=post.community,
@@ -438,6 +449,11 @@ def add_reply(post_id: int, comment_id: int):
     in_reply_to = PostReply.query.get_or_404(comment_id)
     mods = post.community.moderators()
     is_moderator = current_user.is_authenticated and any(mod.user_id == current_user.id for mod in mods)
+    if post.community.private_mods:
+        mod_list = []
+    else:
+        mod_user_ids = [mod.user_id for mod in mods]
+        mod_list = User.query.filter(User.id.in_(mod_user_ids)).all()
 
     if in_reply_to.author.has_blocked_user(current_user.id):
         flash(_('You cannot reply to %(name)s', name=in_reply_to.author.display_name()))
@@ -466,6 +482,10 @@ def add_reply(post_id: int, comment_id: int):
                           body_html=markdown_to_html(form.body.data), body_html_safe=True,
                           from_bot=current_user.bot, nsfw=post.nsfw, nsfl=post.nsfl,
                           notify_author=form.notify_author.data, instance_id=1)
+        if reply.body:
+            for blocked_phrase in blocked_phrases():
+                if blocked_phrase in reply.body:
+                    abort(401)
         db.session.add(reply)
         if in_reply_to.notify_author and current_user.id != in_reply_to.user_id and in_reply_to.author.ap_id is None:    # todo: check if replier is blocked
             notification = Notification(title=shorten_string(_('Reply from %(name)s on %(post_title)s',
@@ -578,7 +598,7 @@ def add_reply(post_id: int, comment_id: int):
         form.notify_author.data = True
         return render_template('post/add_reply.html', title=_('Discussing %(title)s', title=post.title), post=post,
                                is_moderator=is_moderator, form=form, comment=in_reply_to, markdown_editor=current_user.is_authenticated and current_user.markdown_editor,
-                               moderating_communities=moderating_communities(current_user.get_id()),
+                               moderating_communities=moderating_communities(current_user.get_id()), mods=mod_list,
                                joined_communities = joined_communities(current_user.id),
                                inoculation=inoculation[randint(0, len(inoculation) - 1)])
 
@@ -607,6 +627,14 @@ def post_edit(post_id: int):
     post = Post.query.get_or_404(post_id)
     form = CreatePostForm()
     del form.communities
+
+    mods = post.community.moderators()
+    if post.community.private_mods:
+        mod_list = []
+    else:
+        mod_user_ids = [mod.user_id for mod in mods]
+        mod_list = User.query.filter(User.id.in_(mod_user_ids)).all()
+
     if post.user_id == current_user.id or post.community.is_moderator() or current_user.is_admin():
         if g.site.enable_nsfl is False:
             form.nsfl.render_kw = {'disabled': True}
@@ -727,7 +755,7 @@ def post_edit(post_id: int):
             if not (post.community.is_moderator() or post.community.is_owner() or current_user.is_admin()):
                 form.sticky.render_kw = {'disabled': True}
             return render_template('post/post_edit.html', title=_('Edit post'), form=form, post=post,
-                                   markdown_editor=current_user.markdown_editor,
+                                   markdown_editor=current_user.markdown_editor, mods=mod_list,
                                    moderating_communities=moderating_communities(current_user.get_id()),
                                    joined_communities=joined_communities(current_user.get_id()),
                                    inoculation=inoculation[randint(0, len(inoculation) - 1)]

app/utils.py

@@ -330,6 +330,23 @@ def blocked_instances(user_id) -> List[int]:
     return [block.instance_id for block in blocks]
 
 
+@cache.memoize(timeout=86400)
+def blocked_phrases() -> List[str]:
+    site = Site.query.get(1)
+    if site.blocked_phrases:
+        return [phrase for phrase in site.blocked_phrases.split('\n') if phrase != '']
+    else:
+        return []
+
+
+@cache.memoize(timeout=86400)
+def blocked_referrers() -> List[str]:
+    site = Site.query.get(1)
+    if site.auto_decline_referrers:
+        return [referrer for referrer in site.auto_decline_referrers.split('\n') if referrer != '']
+    else:
+        return []
+
 def retrieve_block_list():
     try:
         response = requests.get('https://raw.githubusercontent.com/rimu/no-qanon/master/domains.txt', timeout=1)

migrations/versions/2b028a70bd7a_blocked_phrases.py

@@ -0,0 +1,32 @@
+"""blocked phrases
+
+Revision ID: 2b028a70bd7a
+Revises: 12d60b9d5417
+Create Date: 2024-03-22 11:50:15.405786
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '2b028a70bd7a'
+down_revision = '12d60b9d5417'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('site', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('blocked_phrases', sa.Text(), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('site', schema=None) as batch_op:
+        batch_op.drop_column('blocked_phrases')
+
+    # ### end Alembic commands ###