From 097d37d7e0d7dafc4424a4cb8ed26bfac9c86c35 Mon Sep 17 00:00:00 2001 From: rimu <3310831+rimu@users.noreply.github.com> Date: Wed, 20 Mar 2024 21:19:34 +1300 Subject: [PATCH 01/21] markdown: Parse ~~strikethrough~~ formatting --- app/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/utils.py b/app/utils.py index 4fbcd545..bb775f67 100644 --- a/app/utils.py +++ b/app/utils.py @@ -251,7 +251,7 @@ def html_to_markdown_worker(element, indent_level=0): def markdown_to_html(markdown_text) -> str: if markdown_text: - return allowlist_html(markdown2.markdown(markdown_text, safe_mode=True, extras={'middle-word-em': False, 'tables': True, 'fenced-code-blocks': True, 'spoiler': True})) + return allowlist_html(markdown2.markdown(markdown_text, safe_mode=True, extras={'middle-word-em': False, 'tables': True, 'fenced-code-blocks': True, 'strike': True})) else: return '' From 4a25df2de7843fe93c902936fc2857897283c189 Mon Sep 17 00:00:00 2001 From: freamon Date: Wed, 20 Mar 2024 10:45:26 +0000 Subject: [PATCH 02/21] Indicate Bot Accounts in UI --- app/static/scss/_typography.scss | 8 ++++++++ app/static/structure.css | 8 ++++++++ app/static/styles.css | 8 ++++++++ app/templates/base.html | 3 +++ app/templates/post/continue_discussion.html | 3 +++ app/templates/post/post.html | 3 +++ app/templates/user/show_profile.html | 3 +++ 7 files changed, 36 insertions(+) diff --git a/app/static/scss/_typography.scss b/app/static/scss/_typography.scss index 780e81d2..002d6197 100644 --- a/app/static/scss/_typography.scss +++ b/app/static/scss/_typography.scss @@ -282,6 +282,14 @@ h1 { } } +.fe-bot-account { + position: relative; + top: 1px; + &:before { + content: "\e94d"; + } +} + .fe-video { position: relative; top: 2px; diff --git a/app/static/structure.css b/app/static/structure.css index 0500d613..47e25f42 100644 --- a/app/static/structure.css +++ b/app/static/structure.css @@ -305,6 +305,14 @@ h1 .fe-bell, h1 .fe-no-bell { content: "\e986"; } +.fe-bot-account { + position: relative; + top: 1px; +} +.fe-bot-account:before { + content: "\e94d"; +} + .fe-video { position: relative; top: 2px; diff --git a/app/static/styles.css b/app/static/styles.css index 149e1db0..d9d731ff 100644 --- a/app/static/styles.css +++ b/app/static/styles.css @@ -304,6 +304,14 @@ h1 .fe-bell, h1 .fe-no-bell { content: "\e986"; } +.fe-bot-account { + position: relative; + top: 1px; +} +.fe-bot-account:before { + content: "\e94d"; +} + .fe-video { position: relative; top: 2px; diff --git a/app/templates/base.html b/app/templates/base.html index 879c124f..f7b34d82 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -11,6 +11,9 @@ {% if user.created_recently() %} {% endif %} + {% if user.bot %} + + {% endif %} {% if user.reputation < -10 %} diff --git a/app/templates/post/continue_discussion.html b/app/templates/post/continue_discussion.html index dae25ab5..d0e19b7e 100644 --- a/app/templates/post/continue_discussion.html +++ b/app/templates/post/continue_discussion.html @@ -29,6 +29,9 @@ {% if comment['comment'].author.created_recently() %} {% endif %} + {% if comment['comment'].author.bot %} + + {% endif %} {% if comment['comment'].author.id != current_user.id %} {% if comment['comment'].author.reputation < -10 %} diff --git a/app/templates/post/post.html b/app/templates/post/post.html index 1ba56c72..3aed36a8 100644 --- a/app/templates/post/post.html +++ b/app/templates/post/post.html @@ -90,6 +90,9 @@ {% if comment['comment'].author.created_recently() %} {% endif %} + {% if comment['comment'].author.bot %} + + {% endif %} {% if comment['comment'].author.id != current_user.id %} {% if comment['comment'].author.reputation < -10 %} diff --git a/app/templates/user/show_profile.html b/app/templates/user/show_profile.html index fc880048..17cced75 100644 --- a/app/templates/user/show_profile.html +++ b/app/templates/user/show_profile.html @@ -57,6 +57,9 @@ {% endif %}

{{ _('Joined') }}: {{ moment(user.created).fromNow(refresh=True) }}
+ {% if user.bot %} + {{ _('Bot Account') }}
+ {% endif %} {{ _('Attitude') }}: {{ (user.attitude * 100) | round | int }}%

{{ user.about_html|safe }} {% if posts %} From 82a092eec835d4b44496cad5a4fc4fda46f2606f Mon Sep 17 00:00:00 2001 From: freamon Date: Wed, 20 Mar 2024 10:50:42 +0000 Subject: [PATCH 03/21] Also prevent bot accounts from voting --- app/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/utils.py b/app/utils.py index 4fbcd545..dd1d64cc 100644 --- a/app/utils.py +++ b/app/utils.py @@ -442,7 +442,7 @@ def banned_ip_addresses() -> List[str]: def can_downvote(user, community: Community, site=None) -> bool: - if user is None or community is None or user.banned: + if user is None or community is None or user.banned or user.bot: return False if site is None: @@ -464,7 +464,7 @@ def can_downvote(user, community: Community, site=None) -> bool: def can_upvote(user, community: Community) -> bool: - if user is None or community is None or user.banned: + if user is None or community is None or user.banned or user.bot: return False return True From 0966ce43461cf246dcee4c2e3946958d58594a16 Mon Sep 17 00:00:00 2001 From: freamon Date: Wed, 20 Mar 2024 11:34:25 +0000 Subject: [PATCH 04/21] Recognise 'service' type as bots --- app/activitypub/routes.py | 2 +- app/activitypub/util.py | 3 ++- app/user/utils.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index 1084de75..799fb21e 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -195,7 +195,7 @@ def user_profile(actor): if is_activitypub_request(): server = current_app.config['SERVER_NAME'] actor_data = { "@context": default_context(), - "type": "Person", + "type": "Person" if not user.bot else "Service", "id": f"https://{server}/u/{actor}", "preferredUsername": actor, "name": user.title if user.title else user.user_name, diff --git a/app/activitypub/util.py b/app/activitypub/util.py index a094b19f..a59c5f19 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -488,7 +488,7 @@ def refresh_community_profile_task(community_id): def actor_json_to_model(activity_json, address, server): - if activity_json['type'] == 'Person': + if activity_json['type'] == 'Person' or activity_json['type'] == 'Service': try: user = User(user_name=activity_json['preferredUsername'], title=activity_json['name'] if 'name' in activity_json else None, @@ -508,6 +508,7 @@ def actor_json_to_model(activity_json, address, server): ap_fetched_at=utcnow(), ap_domain=server, public_key=activity_json['publicKey']['publicKeyPem'], + bot=True if activity_json['type'] == 'Service' else False, instance_id=find_instance_id(server) # language=community_json['language'][0]['identifier'] # todo: language ) diff --git a/app/user/utils.py b/app/user/utils.py index 87648d51..f92c5979 100644 --- a/app/user/utils.py +++ b/app/user/utils.py @@ -149,7 +149,7 @@ def search_for_user(address: str): if user_data.status_code == 200: user_json = user_data.json() user_data.close() - if user_json['type'] == 'Person': + if user_json['type'] == 'Person' or user_json['type'] == 'Service': user = actor_json_to_model(user_json, name, server) return user return None From c562202588e5c09cc87d7d214d49a393e1a8cfc6 Mon Sep 17 00:00:00 2001 From: rimu <3310831+rimu@users.noreply.github.com> Date: Thu, 21 Mar 2024 11:07:11 +1300 Subject: [PATCH 05/21] tighten session cookie security --- app/models.py | 4 ++-- config.py | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/app/models.py b/app/models.py index 3a82c5b9..9b2d7a10 100644 --- a/app/models.py +++ b/app/models.py @@ -579,8 +579,8 @@ class User(UserMixin, db.Model): def num_content(self): content = 0 - content += db.session.execute(text('SELECT COUNT(id) as c FROM "post" WHERE user_id = ' + str(self.id))).scalar() - content += db.session.execute(text('SELECT COUNT(id) as c FROM "post_reply" WHERE user_id = ' + str(self.id))).scalar() + content += db.session.execute(text('SELECT COUNT(id) as c FROM "post" WHERE user_id = :user_id'), {'user_id': self.id}).scalar() + content += db.session.execute(text('SELECT COUNT(id) as c FROM "post_reply" WHERE user_id = :user_id'), {'user_id': self.id}).scalar() return content def is_local(self): diff --git a/config.py b/config.py index 77733c73..f26b3fe4 100644 --- a/config.py +++ b/config.py @@ -42,3 +42,7 @@ class Config(object): SENTRY_DSN = os.environ.get('SENTRY_DSN') or None AWS_REGION = os.environ.get('AWS_REGION') or None + + SESSION_COOKIE_SECURE = True + SESSION_COOKIE_HTTPONLY = True + SESSION_COOKIE_SAMESITE = 'Lax' From 9fd4ac1c538e8f20f33793d0c7b853b43364447a Mon Sep 17 00:00:00 2001 From: rimu <3310831+rimu@users.noreply.github.com> Date: Thu, 21 Mar 2024 21:19:50 +1300 Subject: [PATCH 06/21] ban people from community, and unban them #21 --- app/community/forms.py | 2 +- app/community/routes.py | 54 +++++++++++++++++-- app/main/routes.py | 15 +++++- .../community/community_moderate_banned.html | 5 +- app/utils.py | 24 ++++++++- 5 files changed, 90 insertions(+), 10 deletions(-) diff --git a/app/community/forms.py b/app/community/forms.py index bb700ede..5f87147f 100644 --- a/app/community/forms.py +++ b/app/community/forms.py @@ -68,7 +68,7 @@ class SearchRemoteCommunity(FlaskForm): class BanUserCommunityForm(FlaskForm): reason = StringField(_l('Reason'), render_kw={'autofocus': True}, validators=[DataRequired()]) - ban_until = DateField(_l('Ban until')) + ban_until = DateField(_l('Ban until'), validators=[Optional()]) delete_posts = BooleanField(_l('Also delete all their posts')) delete_post_replies = BooleanField(_l('Also delete all their comments')) submit = SubmitField(_l('Ban')) diff --git a/app/community/routes.py b/app/community/routes.py index ed350e22..21f5a69a 100644 --- a/app/community/routes.py +++ b/app/community/routes.py @@ -27,7 +27,7 @@ from app.utils import get_setting, render_template, allowlist_html, markdown_to_ shorten_string, gibberish, community_membership, ap_datetime, \ request_etag_matches, return_304, instance_banned, can_create_post, can_upvote, can_downvote, user_filters_posts, \ joined_communities, moderating_communities, blocked_domains, mimetype_from_url, blocked_instances, \ - community_moderators + community_moderators, communities_banned_from from feedgen.feed import FeedGenerator from datetime import timezone, timedelta @@ -793,6 +793,9 @@ def community_ban_user(community_id: int, user_id: int): form = BanUserCommunityForm() if form.validate_on_submit(): + # Both CommunityBan and CommunityMember need to be updated. CommunityBan is under the control of moderators while + # CommunityMember can be cleared by the user by leaving the group and rejoining. CommunityMember.is_banned stops + # posts from the community from showing up in the banned person's home feed. if not existing: new_ban = CommunityBan(community_id=community_id, user_id=user.id, banned_by=current_user.id, reason=form.reason.data) @@ -800,7 +803,13 @@ def community_ban_user(community_id: int, user_id: int): new_ban.ban_until = form.ban_until.data db.session.add(new_ban) db.session.commit() - flash(_('%(name)s has been banned.', name=user.display_name())) + + community_membership_record = CommunityMember.query.filter_by(community_id=community.id, user_id=user.id).first() + if community_membership_record: + community_membership_record.is_banned = True + db.session.commit() + + flash(_('%(name)s has been banned.', name=user.display_name())) if form.delete_posts.data: posts = Post.query.filter(Post.user_id == user.id, Post.community_id == community.id).all() @@ -819,8 +828,11 @@ def community_ban_user(community_id: int, user_id: int): # notify banned person if user.is_local(): + cache.delete_memoized(communities_banned_from, user.id) + cache.delete_memoized(joined_communities, user.id) + cache.delete_memoized(moderating_communities, user.id) notify = Notification(title=shorten_string('You have been banned from ' + community.title), - url=f'/', user_id=user.id, + url=f'/notifications', user_id=user.id, author_id=1) db.session.add(notify) user.unread_notifications += 1 @@ -839,6 +851,42 @@ def community_ban_user(community_id: int, user_id: int): ) +@bp.route('/community///unban_user_community', methods=['GET', 'POST']) +@login_required +def community_unban_user(community_id: int, user_id: int): + community = Community.query.get_or_404(community_id) + user = User.query.get_or_404(user_id) + existing_ban = CommunityBan.query.filter_by(community_id=community.id, user_id=user.id).first() + if existing_ban: + db.session.delete(existing_ban) + db.session.commit() + + community_membership_record = CommunityMember.query.filter_by(community_id=community.id, user_id=user.id).first() + if community_membership_record: + community_membership_record.is_banned = False + db.session.commit() + + flash(_('%(name)s has been unbanned.', name=user.display_name())) + + # todo: federate ban to post author instance + + # notify banned person + if user.is_local(): + cache.delete_memoized(communities_banned_from, user.id) + cache.delete_memoized(joined_communities, user.id) + cache.delete_memoized(moderating_communities, user.id) + notify = Notification(title=shorten_string('You have been un-banned from ' + community.title), + url=f'/notifications', user_id=user.id, + author_id=1) + db.session.add(notify) + user.unread_notifications += 1 + db.session.commit() + else: + ... + # todo: send chatmessage to remote user and federate it + + return redirect(url_for('community.community_moderate_banned', actor=community.link())) + @bp.route('//notification', methods=['GET', 'POST']) @login_required diff --git a/app/main/routes.py b/app/main/routes.py index b2d292a0..44cca30e 100644 --- a/app/main/routes.py +++ b/app/main/routes.py @@ -24,7 +24,7 @@ from sqlalchemy_searchable import search from app.utils import render_template, get_setting, gibberish, request_etag_matches, return_304, blocked_domains, \ ap_datetime, ip_address, retrieve_block_list, shorten_string, markdown_to_text, user_filters_home, \ joined_communities, moderating_communities, parse_page, theme_list, get_request, markdown_to_html, allowlist_html, \ - blocked_instances + blocked_instances, communities_banned_from from app.models import Community, CommunityMember, Post, Site, User, utcnow, Domain, Topic, File, Instance, \ InstanceRole, Notification from PIL import Image @@ -131,7 +131,13 @@ def home_page(type, sort): next_url = url_for('main.all_posts', page=posts.next_num, sort=sort) if posts.has_next else None prev_url = url_for('main.all_posts', page=posts.prev_num, sort=sort) if posts.has_prev and page != 1 else None - active_communities = Community.query.filter_by(banned=False).order_by(desc(Community.last_active)).limit(5).all() + # Active Communities + active_communities = Community.query.filter_by(banned=False) + if current_user.is_authenticated: # do not show communities current user is banned from + banned_from = communities_banned_from(current_user.id) + if banned_from: + active_communities = active_communities.filter(Community.id.not_in(banned_from)) + active_communities = active_communities.order_by(desc(Community.last_active)).limit(5).all() return render_template('index.html', posts=posts, active_communities=active_communities, show_post_community=True, POST_TYPE_IMAGE=POST_TYPE_IMAGE, POST_TYPE_LINK=POST_TYPE_LINK, @@ -178,6 +184,11 @@ def list_communities(): if topic_id != 0: communities = communities.filter_by(topic_id=topic_id) + if current_user.is_authenticated: + banned_from = communities_banned_from(current_user.id) + if banned_from: + communities = communities.filter(Community.id.not_in(banned_from)) + return render_template('list_communities.html', communities=communities.order_by(sort_by).all(), search=search_param, title=_('Communities'), SUBSCRIPTION_PENDING=SUBSCRIPTION_PENDING, SUBSCRIPTION_MEMBER=SUBSCRIPTION_MEMBER, SUBSCRIPTION_OWNER=SUBSCRIPTION_OWNER, SUBSCRIPTION_MODERATOR=SUBSCRIPTION_MODERATOR, diff --git a/app/templates/community/community_moderate_banned.html b/app/templates/community/community_moderate_banned.html index 3ce5a8f2..96505f46 100644 --- a/app/templates/community/community_moderate_banned.html +++ b/app/templates/community/community_moderate_banned.html @@ -48,11 +48,12 @@ {{ user.reports if user.reports > 0 }} {{ user.ip_address if user.ip_address }} {% if user.is_local() %} - View local + View {% else %} + View local | View remote {% endif %} - | {{ _('Un ban') }} + | {{ _('Un ban') }} {% endfor %} diff --git a/app/utils.py b/app/utils.py index bb775f67..afffa592 100644 --- a/app/utils.py +++ b/app/utils.py @@ -28,7 +28,7 @@ import re from app.email import send_welcome_email from app.models import Settings, Domain, Instance, BannedInstances, User, Community, DomainBlock, ActivityPubLog, IpBan, \ - Site, Post, PostReply, utcnow, Filter, CommunityMember, InstanceBlock + Site, Post, PostReply, utcnow, Filter, CommunityMember, InstanceBlock, CommunityBan # Flask's render_template function, with support for themes added @@ -312,6 +312,12 @@ def community_membership(user: User, community: Community) -> int: return user.subscribed(community.id) +@cache.memoize(timeout=86400) +def communities_banned_from(user_id) -> List[int]: + community_bans = CommunityBan.query.filter(CommunityBan.user_id == user_id).all() + return [cb.community_id for cb in community_bans] + + @cache.memoize(timeout=86400) def blocked_domains(user_id) -> List[int]: blocks = DomainBlock.query.filter_by(user_id=user_id) @@ -460,6 +466,9 @@ def can_downvote(user, community: Community, site=None) -> bool: if user.attitude < -0.40 or user.reputation < -10: # this should exclude about 3.7% of users. return False + if community.id in communities_banned_from(user.id): + return False + return True @@ -467,6 +476,9 @@ def can_upvote(user, community: Community) -> bool: if user is None or community is None or user.banned: return False + if community.id in communities_banned_from(user.id): + return False + return True @@ -483,6 +495,9 @@ def can_create_post(user, content: Community) -> bool: if content.local_only and not user.is_local(): return False + if content.id in communities_banned_from(user.id): + return False + return True @@ -496,6 +511,9 @@ def can_create_post_reply(user, content: Community) -> bool: if content.local_only and not user.is_local(): return False + if content.id in communities_banned_from(user.id): + return False + return True @@ -602,7 +620,8 @@ def moderating_communities(user_id): return [] return Community.query.join(CommunityMember, Community.id == CommunityMember.community_id).\ filter(Community.banned == False).\ - filter(or_(CommunityMember.is_moderator == True, CommunityMember.is_owner == True)).\ + filter(or_(CommunityMember.is_moderator == True, CommunityMember.is_owner == True)). \ + filter(CommunityMember.is_banned == False). \ filter(CommunityMember.user_id == user_id).order_by(Community.title).all() @@ -613,6 +632,7 @@ def joined_communities(user_id): return Community.query.join(CommunityMember, Community.id == CommunityMember.community_id).\ filter(Community.banned == False). \ filter(CommunityMember.is_moderator == False, CommunityMember.is_owner == False). \ + filter(CommunityMember.is_banned == False). \ filter(CommunityMember.user_id == user_id).order_by(Community.title).all() From 1a8c21670e48922653933013f558097f73875c58 Mon Sep 17 00:00:00 2001 From: rimu <3310831+rimu@users.noreply.github.com> Date: Thu, 21 Mar 2024 21:24:30 +1300 Subject: [PATCH 07/21] ban people from community, and unban them - topics and search #21 --- app/search/routes.py | 8 ++++++-- app/topic/routes.py | 6 +++++- app/user/routes.py | 2 +- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/app/search/routes.py b/app/search/routes.py index 8edbcef7..654be6fe 100644 --- a/app/search/routes.py +++ b/app/search/routes.py @@ -5,7 +5,8 @@ from sqlalchemy import or_ from app.models import Post from app.search import bp -from app.utils import moderating_communities, joined_communities, render_template, blocked_domains, blocked_instances +from app.utils import moderating_communities, joined_communities, render_template, blocked_domains, blocked_instances, \ + communities_banned_from @bp.route('/search', methods=['GET', 'POST']) @@ -29,6 +30,9 @@ def run_search(): instance_ids = blocked_instances(current_user.id) if instance_ids: posts = posts.filter(or_(Post.instance_id.not_in(instance_ids), Post.instance_id == None)) + banned_from = communities_banned_from(current_user.id) + if banned_from: + posts = posts.filter(Post.community_id.not_in(banned_from)) else: posts = posts.filter(Post.from_bot == False) posts = posts.filter(Post.nsfl == False) @@ -43,7 +47,7 @@ def run_search(): prev_url = url_for('search.run_search', page=posts.prev_num, q=q) if posts.has_prev and page != 1 else None return render_template('search/results.html', title=_('Search results for %(q)s', q=q), posts=posts, q=q, - next_url=next_url, prev_url=prev_url, + next_url=next_url, prev_url=prev_url, show_post_community=True, moderating_communities=moderating_communities(current_user.get_id()), joined_communities=joined_communities(current_user.get_id()), site=g.site) diff --git a/app/topic/routes.py b/app/topic/routes.py index 906ec7ee..2e258757 100644 --- a/app/topic/routes.py +++ b/app/topic/routes.py @@ -16,7 +16,8 @@ from app.topic import bp from app import db, celery, cache from app.topic.forms import ChooseTopicsForm from app.utils import render_template, user_filters_posts, moderating_communities, joined_communities, \ - community_membership, blocked_domains, validation_required, mimetype_from_url, blocked_instances + community_membership, blocked_domains, validation_required, mimetype_from_url, blocked_instances, \ + communities_banned_from @bp.route('/topic/', methods=['GET']) @@ -68,6 +69,9 @@ def show_topic(topic_path): instance_ids = blocked_instances(current_user.id) if instance_ids: posts = posts.filter(or_(Post.instance_id.not_in(instance_ids), Post.instance_id == None)) + banned_from = communities_banned_from(current_user.id) + if banned_from: + posts = posts.filter(Post.community_id.not_in(banned_from)) # sorting if sort == '' or sort == 'hot': diff --git a/app/user/routes.py b/app/user/routes.py index bbaa1444..91ba8cb7 100644 --- a/app/user/routes.py +++ b/app/user/routes.py @@ -79,7 +79,7 @@ def show_profile(user): description=description, subscribed=subscribed, upvoted=upvoted, post_next_url=post_next_url, post_prev_url=post_prev_url, replies_next_url=replies_next_url, replies_prev_url=replies_prev_url, - noindex=not user.indexable, + noindex=not user.indexable, show_post_community=True, moderating_communities=moderating_communities(current_user.get_id()), joined_communities=joined_communities(current_user.get_id()) ) From 43f09983917519a0ca3b34fd2bf16b5bc461c828 Mon Sep 17 00:00:00 2001 From: rimu <3310831+rimu@users.noreply.github.com> Date: Thu, 21 Mar 2024 21:50:25 +1300 Subject: [PATCH 08/21] When a post or postreply is created, record whether the author is a bot, for easier filtering #98 #114 --- app/activitypub/util.py | 2 ++ app/post/util.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/app/activitypub/util.py b/app/activitypub/util.py index a59c5f19..b60e7d8c 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -1159,6 +1159,7 @@ def create_post_reply(activity_log: ActivityPubLog, community: Community, in_rep root_id=root_id, nsfw=community.nsfw, nsfl=community.nsfl, + from_bot=user.bot, up_votes=1, depth=depth, score=instance_weight(user.ap_domain), @@ -1257,6 +1258,7 @@ def create_post(activity_log: ActivityPubLog, community: Community, request_json ap_announce_id=announce_id, type=constants.POST_TYPE_ARTICLE, up_votes=1, + from_bot=user.bot, score=instance_weight(user.ap_domain), instance_id=user.instance_id, indexable=user.indexable diff --git a/app/post/util.py b/app/post/util.py index 948e3f1f..99e7f6cb 100644 --- a/app/post/util.py +++ b/app/post/util.py @@ -15,6 +15,8 @@ def post_replies(post_id: int, sort_by: str, show_first: int = 0) -> List[PostRe instance_ids = blocked_instances(current_user.id) if instance_ids: comments = comments.filter(or_(PostReply.instance_id.not_in(instance_ids), PostReply.instance_id == None)) + if current_user.ignore_bots: + comments = comments.filter(PostReply.from_bot == False) if sort_by == 'hot': comments = comments.order_by(desc(PostReply.ranking)) elif sort_by == 'top': From 2cbaefd844a3f2a8c63fe0f1b733c223752d102c Mon Sep 17 00:00:00 2001 From: rimu <3310831+rimu@users.noreply.github.com> Date: Fri, 22 Mar 2024 10:50:17 +1300 Subject: [PATCH 09/21] block shitpost flood --- app/activitypub/util.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/activitypub/util.py b/app/activitypub/util.py index b60e7d8c..85a92e53 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -1177,6 +1177,9 @@ def create_post_reply(activity_log: ActivityPubLog, community: Community, in_rep post_reply.body_html = allowlist_html(request_json['object']['content']) post_reply.body = html_to_markdown(post_reply.body_html) if post_id is not None: + # block shitpost flood + if post_reply.body and "SNEED'S" in post_reply.body: + return None post = Post.query.get(post_id) if post.comments_enabled: anchor = None From f7074d2edde05cb0baaaeb1f038f79a700c0b63b Mon Sep 17 00:00:00 2001 From: freamon Date: Thu, 21 Mar 2024 23:21:28 +0000 Subject: [PATCH 10/21] Add/Remove moderators (incoming AP) --- app/activitypub/routes.py | 44 ++++++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index 799fb21e..922c2385 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -666,24 +666,48 @@ def process_inbox_request(request_json, activitypublog_id, ip_address): target_ap_id = request_json['object']['object']['object'] # object object object! post = undo_vote(activity_log, comment, post, target_ap_id, user) activity_log.result = 'success' - elif request_json['object']['type'] == 'Add': + elif request_json['object']['type'] == 'Add' and 'target' in request_json['object']: activity_log.activity_type = request_json['object']['type'] - featured_url = Community.query.filter(Community.ap_public_url == request_json['actor']).first().ap_featured_url - if featured_url: - if 'target' in request_json['object'] and featured_url == request_json['object']['target']: - post = Post.query.filter(Post.ap_id == request_json['object']['object']).first() + target = request_json['object']['target'] + community = Community.query.filter_by(ap_public_url=request_json['actor']).first() + if community: + featured_url = community.ap_featured_url + moderators_url = community.ap_moderators_url + if target == featured_url: + post = Post.query.filter_by(ap_id=request_json['object']['object']).first() if post: post.sticky = True activity_log.result = 'success' - elif request_json['object']['type'] == 'Remove': + if target == moderators_url: + user = find_actor_or_create(request_json['object']['object']) + if user: + existing_membership = CommunityMember.query.filter_by(community_id=community.id, user_id=user.id).first() + if existing_membership: + existing_membership.is_moderator = True + else: + new_membership = CommunityMember(community_id=community.id, user_id=user.id, is_moderator=True) + db.session.add(new_membership) + db.session.commit() + activity_log.result = 'success' + elif request_json['object']['type'] == 'Remove' and 'target' in request_json['object']: activity_log.activity_type = request_json['object']['type'] - featured_url = Community.query.filter(Community.ap_public_url == request_json['actor']).first().ap_featured_url - if featured_url: - if 'target' in request_json['object'] and featured_url == request_json['object']['target']: - post = Post.query.filter(Post.ap_id == request_json['object']['object']).first() + target = request_json['object']['target'] + community = Community.query.filter_by(ap_public_url=request_json['actor']).first() + if community: + featured_url = community.ap_featured_url + moderators_url = community.ap_moderators_url + if target == featured_url: + post = Post.query.filter_by(ap_id=request_json['object']['object']).first() if post: post.sticky = False activity_log.result = 'success' + if target == moderators_url: + user = find_actor_or_create(request_json['object']['object'], create_if_not_found=False) + if user: + existing_membership = CommunityMember.query.filter_by(community_id=community.id, user_id=user.id).first() + if existing_membership: + existing_membership.is_moderator = False + activity_log.result = 'success' else: activity_log.exception_message = 'Invalid type for Announce' From 081108a7c6aea74c05d1ce232c9b2bf73aa1bcf9 Mon Sep 17 00:00:00 2001 From: rimu <3310831+rimu@users.noreply.github.com> Date: Fri, 22 Mar 2024 12:22:19 +1300 Subject: [PATCH 11/21] block future shitpost floods --- app/activitypub/util.py | 20 +++++++++--- app/admin/forms.py | 1 + app/admin/routes.py | 12 +++++-- app/community/util.py | 15 ++++++++- app/models.py | 3 +- app/post/routes.py | 6 +++- app/utils.py | 9 ++++++ .../versions/2b028a70bd7a_blocked_phrases.py | 32 +++++++++++++++++++ 8 files changed, 88 insertions(+), 10 deletions(-) create mode 100644 migrations/versions/2b028a70bd7a_blocked_phrases.py diff --git a/app/activitypub/util.py b/app/activitypub/util.py index 85a92e53..239dedcf 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -24,7 +24,8 @@ import pytesseract from app.utils import get_request, allowlist_html, html_to_markdown, get_setting, ap_datetime, markdown_to_html, \ is_image_url, domain_from_url, gibberish, ensure_directory_exists, markdown_to_text, head_request, post_ranking, \ - shorten_string, reply_already_exists, reply_is_just_link_to_gif_reaction, confidence, remove_tracking_from_link + shorten_string, reply_already_exists, reply_is_just_link_to_gif_reaction, confidence, remove_tracking_from_link, \ + blocked_phrases def public_key(): @@ -1177,9 +1178,11 @@ def create_post_reply(activity_log: ActivityPubLog, community: Community, in_rep post_reply.body_html = allowlist_html(request_json['object']['content']) post_reply.body = html_to_markdown(post_reply.body_html) if post_id is not None: - # block shitpost flood - if post_reply.body and "SNEED'S" in post_reply.body: - return None + # Discard post_reply if it contains certain phrases. Good for stopping spam floods. + if post_reply.body: + for blocked_phrase in blocked_phrases(): + if blocked_phrase in post_reply.body: + return None post = Post.query.get(post_id) if post.comments_enabled: anchor = None @@ -1273,6 +1276,15 @@ def create_post(activity_log: ActivityPubLog, community: Community, request_json elif 'content' in request_json['object'] and request_json['object']['content'] is not None: # Kbin post.body_html = allowlist_html(request_json['object']['content']) post.body = html_to_markdown(post.body_html) + # Discard post if it contains certain phrases. Good for stopping spam floods. + blocked_phrases_list = blocked_phrases() + for blocked_phrase in blocked_phrases_list: + if blocked_phrase in post.title: + return None + if post.body: + for blocked_phrase in blocked_phrases_list: + if blocked_phrase in post.body: + return None if 'attachment' in request_json['object'] and len(request_json['object']['attachment']) > 0 and \ 'type' in request_json['object']['attachment'][0]: if request_json['object']['attachment'][0]['type'] == 'Link': diff --git a/app/admin/forms.py b/app/admin/forms.py index b0b8ff2c..433f224c 100644 --- a/app/admin/forms.py +++ b/app/admin/forms.py @@ -41,6 +41,7 @@ class FederationForm(FlaskForm): allowlist = TextAreaField(_l('Allow federation with these instances')) use_blocklist = BooleanField(_l('Blocklist instead of allowlist')) blocklist = TextAreaField(_l('Deny federation with these instances')) + blocked_phrases = TextAreaField(_l('Discard all posts and comments with these phrases (one per line)')) submit = SubmitField(_l('Save')) diff --git a/app/admin/routes.py b/app/admin/routes.py index a53d987d..a8068bb4 100644 --- a/app/admin/routes.py +++ b/app/admin/routes.py @@ -6,10 +6,10 @@ from flask_login import login_required, current_user from flask_babel import _ from sqlalchemy import text, desc -from app import db, celery +from app import db, celery, cache from app.activitypub.routes import process_inbox_request, process_delete_request from app.activitypub.signature import post_request -from app.activitypub.util import default_context +from app.activitypub.util import default_context, instance_allowed, instance_blocked from app.admin.forms import FederationForm, SiteMiscForm, SiteProfileForm, EditCommunityForm, EditUserForm, \ EditTopicForm, SendNewsletterForm, AddUserForm from app.admin.util import unsubscribe_from_everything_then_delete, unsubscribe_from_community, send_newsletter, \ @@ -18,7 +18,7 @@ from app.community.util import save_icon_file, save_banner_file from app.models import AllowedInstances, BannedInstances, ActivityPubLog, utcnow, Site, Community, CommunityMember, \ User, Instance, File, Report, Topic, UserRegistration, Role, Post from app.utils import render_template, permission_required, set_setting, get_setting, gibberish, markdown_to_html, \ - moderating_communities, joined_communities, finalize_user_setup, theme_list + moderating_communities, joined_communities, finalize_user_setup, theme_list, blocked_phrases from app.admin import bp @@ -123,13 +123,18 @@ def admin_federation(): for allow in form.allowlist.data.split('\n'): if allow.strip(): db.session.add(AllowedInstances(domain=allow.strip())) + cache.delete_memoized(instance_allowed, allow.strip()) if form.use_blocklist.data: set_setting('use_allowlist', False) db.session.execute(text('DELETE FROM banned_instances')) for banned in form.blocklist.data.split('\n'): if banned.strip(): db.session.add(BannedInstances(domain=banned.strip())) + cache.delete_memoized(instance_blocked, banned.strip()) + site.blocked_phrases = form.blocked_phrases.data + cache.delete_memoized(blocked_phrases) db.session.commit() + flash(_('Admin settings saved')) elif request.method == 'GET': @@ -139,6 +144,7 @@ def admin_federation(): form.blocklist.data = '\n'.join([instance.domain for instance in instances]) instances = AllowedInstances.query.all() form.allowlist.data = '\n'.join([instance.domain for instance in instances]) + form.blocked_phrases.data = site.blocked_phrases return render_template('admin/federation.html', title=_('Federation settings'), form=form, moderating_communities=moderating_communities(current_user.get_id()), diff --git a/app/community/util.py b/app/community/util.py index aedb3615..758f0bd3 100644 --- a/app/community/util.py +++ b/app/community/util.py @@ -16,7 +16,7 @@ from app.models import Community, File, BannedInstances, PostReply, PostVote, Po Instance, Notification, User, ActivityPubLog from app.utils import get_request, gibberish, markdown_to_html, domain_from_url, allowlist_html, \ html_to_markdown, is_image_url, ensure_directory_exists, inbox_domain, post_ranking, shorten_string, parse_page, \ - remove_tracking_from_link, ap_datetime, instance_banned + remove_tracking_from_link, ap_datetime, instance_banned, blocked_phrases from sqlalchemy import func, desc import os @@ -299,6 +299,19 @@ def save_post(form, post: Post): if current_user.reputation < -100: post.score = -1 post.ranking = post_ranking(post.score, utcnow()) + + # Filter by phrase + blocked_phrases_list = blocked_phrases() + for blocked_phrase in blocked_phrases_list: + if blocked_phrase in post.title: + abort(401) + return + if post.body: + for blocked_phrase in blocked_phrases_list: + if blocked_phrase in post.body: + abort(401) + return + db.session.add(post) g.site.last_active = utcnow() diff --git a/app/models.py b/app/models.py index 9b2d7a10..5c0c4eb3 100644 --- a/app/models.py +++ b/app/models.py @@ -1170,7 +1170,8 @@ class Site(db.Model): allow_or_block_list = db.Column(db.Integer, default=2) # 1 = allow list, 2 = block list allowlist = db.Column(db.Text, default='') blocklist = db.Column(db.Text, default='') - auto_decline_referrers = db.Column(db.Text, default='rdrama.net') + blocked_phrases = db.Column(db.Text, default='') # discard incoming content with these phrases + auto_decline_referrers = db.Column(db.Text, default='rdrama.net') # automatically decline registration requests if the referrer is one of these created_at = db.Column(db.DateTime, default=utcnow) updated = db.Column(db.DateTime, default=utcnow) last_active = db.Column(db.DateTime, default=utcnow) diff --git a/app/post/routes.py b/app/post/routes.py index f506c68f..14990cbe 100644 --- a/app/post/routes.py +++ b/app/post/routes.py @@ -24,7 +24,7 @@ from app.utils import get_setting, render_template, allowlist_html, markdown_to_ shorten_string, markdown_to_text, gibberish, ap_datetime, return_304, \ request_etag_matches, ip_address, user_ip_banned, instance_banned, can_downvote, can_upvote, post_ranking, \ reply_already_exists, reply_is_just_link_to_gif_reaction, confidence, moderating_communities, joined_communities, \ - blocked_instances, blocked_domains, community_moderators + blocked_instances, blocked_domains, community_moderators, blocked_phrases def show_post(post_id: int): @@ -466,6 +466,10 @@ def add_reply(post_id: int, comment_id: int): body_html=markdown_to_html(form.body.data), body_html_safe=True, from_bot=current_user.bot, nsfw=post.nsfw, nsfl=post.nsfl, notify_author=form.notify_author.data, instance_id=1) + if reply.body: + for blocked_phrase in blocked_phrases(): + if blocked_phrase in reply.body: + abort(401) db.session.add(reply) if in_reply_to.notify_author and current_user.id != in_reply_to.user_id and in_reply_to.author.ap_id is None: # todo: check if replier is blocked notification = Notification(title=shorten_string(_('Reply from %(name)s on %(post_title)s', diff --git a/app/utils.py b/app/utils.py index 84c462e3..aef7ac30 100644 --- a/app/utils.py +++ b/app/utils.py @@ -330,6 +330,15 @@ def blocked_instances(user_id) -> List[int]: return [block.instance_id for block in blocks] +@cache.memoize(timeout=86400) +def blocked_phrases() -> List[str]: + site = Site.query.get(1) + if site.blocked_phrases: + return [phrase for phrase in site.blocked_phrases.split('\n') if phrase != ''] + else: + return [] + + def retrieve_block_list(): try: response = requests.get('https://raw.githubusercontent.com/rimu/no-qanon/master/domains.txt', timeout=1) diff --git a/migrations/versions/2b028a70bd7a_blocked_phrases.py b/migrations/versions/2b028a70bd7a_blocked_phrases.py new file mode 100644 index 00000000..0f987347 --- /dev/null +++ b/migrations/versions/2b028a70bd7a_blocked_phrases.py @@ -0,0 +1,32 @@ +"""blocked phrases + +Revision ID: 2b028a70bd7a +Revises: 12d60b9d5417 +Create Date: 2024-03-22 11:50:15.405786 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '2b028a70bd7a' +down_revision = '12d60b9d5417' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('site', schema=None) as batch_op: + batch_op.add_column(sa.Column('blocked_phrases', sa.Text(), nullable=True)) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('site', schema=None) as batch_op: + batch_op.drop_column('blocked_phrases') + + # ### end Alembic commands ### From 4804c4c4b2456389d0399fd6852aeaf093309af8 Mon Sep 17 00:00:00 2001 From: freamon Date: Thu, 21 Mar 2024 23:26:03 +0000 Subject: [PATCH 12/21] Avoid returning 'null' for manuallyApprovesFollowers --- app/activitypub/routes.py | 2 +- app/models.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index 922c2385..6bd0a8cf 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -203,7 +203,7 @@ def user_profile(actor): "outbox": f"https://{server}/u/{actor}/outbox", "discoverable": user.searchable, "indexable": user.indexable, - "manuallyApprovesFollowers": user.ap_manually_approves_followers, + "manuallyApprovesFollowers": False if not user.ap_manually_approves_followers else user.ap_manually_approves_followers, "publicKey": { "id": f"https://{server}/u/{actor}#main-key", "owner": f"https://{server}/u/{actor}", diff --git a/app/models.py b/app/models.py index 9b2d7a10..93e0c1fe 100644 --- a/app/models.py +++ b/app/models.py @@ -488,7 +488,7 @@ class User(UserMixin, db.Model): ap_fetched_at = db.Column(db.DateTime) ap_followers_url = db.Column(db.String(255)) ap_preferred_username = db.Column(db.String(255)) - ap_manually_approves_followers = db.Column(db.Boolean) + ap_manually_approves_followers = db.Column(db.Boolean, default=False) ap_deleted_at = db.Column(db.DateTime) ap_inbox_url = db.Column(db.String(255)) ap_domain = db.Column(db.String(255)) From 75c3c0b49b8cc03b4600290c900b3a4c6e80376b Mon Sep 17 00:00:00 2001 From: freamon Date: Fri, 22 Mar 2024 01:23:26 +0000 Subject: [PATCH 13/21] mod_list in side-panel for show_post, add_reply, etc --- app/post/routes.py | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/app/post/routes.py b/app/post/routes.py index f506c68f..df58cf9b 100644 --- a/app/post/routes.py +++ b/app/post/routes.py @@ -18,7 +18,7 @@ from app.post.util import post_replies, get_comment_branch, post_reply_count from app.constants import SUBSCRIPTION_MEMBER, POST_TYPE_LINK, POST_TYPE_IMAGE from app.models import Post, PostReply, \ PostReplyVote, PostVote, Notification, utcnow, UserBlock, DomainBlock, InstanceBlock, Report, Site, Community, \ - Topic + Topic, User from app.post import bp from app.utils import get_setting, render_template, allowlist_html, markdown_to_html, validation_required, \ shorten_string, markdown_to_text, gibberish, ap_datetime, return_304, \ @@ -47,6 +47,12 @@ def show_post(post_id: int): mods = community_moderators(community.id) is_moderator = current_user.is_authenticated and any(mod.user_id == current_user.id for mod in mods) + if community.private_mods: + mod_list = [] + else: + mod_user_ids = [mod.user_id for mod in mods] + mod_list = User.query.filter(User.id.in_(mod_user_ids)).all() + # handle top-level comments/replies form = NewReplyForm() if current_user.is_authenticated and current_user.verified and form.validate_on_submit(): @@ -213,7 +219,7 @@ def show_post(post_id: int): breadcrumbs.append(breadcrumb) response = render_template('post/post.html', title=post.title, post=post, is_moderator=is_moderator, community=post.community, - breadcrumbs=breadcrumbs, related_communities=related_communities, + breadcrumbs=breadcrumbs, related_communities=related_communities, mods=mod_list, canonical=post.ap_id, form=form, replies=replies, THREAD_CUTOFF_DEPTH=constants.THREAD_CUTOFF_DEPTH, description=description, og_image=og_image, POST_TYPE_IMAGE=constants.POST_TYPE_IMAGE, POST_TYPE_LINK=constants.POST_TYPE_LINK, POST_TYPE_ARTICLE=constants.POST_TYPE_ARTICLE, @@ -409,9 +415,14 @@ def continue_discussion(post_id, comment_id): abort(404) mods = post.community.moderators() is_moderator = current_user.is_authenticated and any(mod.user_id == current_user.id for mod in mods) + if post.community.private_mods: + mod_list = [] + else: + mod_user_ids = [mod.user_id for mod in mods] + mod_list = User.query.filter(User.id.in_(mod_user_ids)).all() replies = get_comment_branch(post.id, comment.id, 'top') - response = render_template('post/continue_discussion.html', title=_('Discussing %(title)s', title=post.title), post=post, + response = render_template('post/continue_discussion.html', title=_('Discussing %(title)s', title=post.title), post=post, mods=mod_list, is_moderator=is_moderator, comment=comment, replies=replies, markdown_editor=current_user.is_authenticated and current_user.markdown_editor, moderating_communities=moderating_communities(current_user.get_id()), joined_communities=joined_communities(current_user.get_id()), community=post.community, @@ -438,6 +449,11 @@ def add_reply(post_id: int, comment_id: int): in_reply_to = PostReply.query.get_or_404(comment_id) mods = post.community.moderators() is_moderator = current_user.is_authenticated and any(mod.user_id == current_user.id for mod in mods) + if post.community.private_mods: + mod_list = [] + else: + mod_user_ids = [mod.user_id for mod in mods] + mod_list = User.query.filter(User.id.in_(mod_user_ids)).all() if in_reply_to.author.has_blocked_user(current_user.id): flash(_('You cannot reply to %(name)s', name=in_reply_to.author.display_name())) @@ -578,7 +594,7 @@ def add_reply(post_id: int, comment_id: int): form.notify_author.data = True return render_template('post/add_reply.html', title=_('Discussing %(title)s', title=post.title), post=post, is_moderator=is_moderator, form=form, comment=in_reply_to, markdown_editor=current_user.is_authenticated and current_user.markdown_editor, - moderating_communities=moderating_communities(current_user.get_id()), + moderating_communities=moderating_communities(current_user.get_id()), mods=mod_list, joined_communities = joined_communities(current_user.id), inoculation=inoculation[randint(0, len(inoculation) - 1)]) @@ -607,6 +623,14 @@ def post_edit(post_id: int): post = Post.query.get_or_404(post_id) form = CreatePostForm() del form.communities + + mods = post.community.moderators() + if post.community.private_mods: + mod_list = [] + else: + mod_user_ids = [mod.user_id for mod in mods] + mod_list = User.query.filter(User.id.in_(mod_user_ids)).all() + if post.user_id == current_user.id or post.community.is_moderator() or current_user.is_admin(): if g.site.enable_nsfl is False: form.nsfl.render_kw = {'disabled': True} @@ -727,7 +751,7 @@ def post_edit(post_id: int): if not (post.community.is_moderator() or post.community.is_owner() or current_user.is_admin()): form.sticky.render_kw = {'disabled': True} return render_template('post/post_edit.html', title=_('Edit post'), form=form, post=post, - markdown_editor=current_user.markdown_editor, + markdown_editor=current_user.markdown_editor, mods=mod_list, moderating_communities=moderating_communities(current_user.get_id()), joined_communities=joined_communities(current_user.get_id()), inoculation=inoculation[randint(0, len(inoculation) - 1)] From ef43e78ae5b64d56e00e05659ea77d5020794bb3 Mon Sep 17 00:00:00 2001 From: rimu <3310831+rimu@users.noreply.github.com> Date: Fri, 22 Mar 2024 14:35:51 +1300 Subject: [PATCH 14/21] automatically block new user registrations based on referrer --- app/admin/forms.py | 1 + app/admin/routes.py | 5 ++++- app/auth/routes.py | 7 ++++++- app/models.py | 2 +- app/utils.py | 8 ++++++++ 5 files changed, 20 insertions(+), 3 deletions(-) diff --git a/app/admin/forms.py b/app/admin/forms.py index 433f224c..c47280f9 100644 --- a/app/admin/forms.py +++ b/app/admin/forms.py @@ -31,6 +31,7 @@ class SiteMiscForm(FlaskForm): types = [('Open', _l('Open')), ('RequireApplication', _l('Require application')), ('Closed', _l('Closed'))] registration_mode = SelectField(_l('Registration mode'), choices=types, default=1, coerce=str) application_question = TextAreaField(_l('Question to ask people applying for an account')) + auto_decline_referrers = TextAreaField(_l('Block registrations from these referrers (one per line)')) log_activitypub_json = BooleanField(_l('Log ActivityPub JSON for debugging')) default_theme = SelectField(_l('Default theme'), coerce=str) submit = SubmitField(_l('Save')) diff --git a/app/admin/routes.py b/app/admin/routes.py index a8068bb4..776d1169 100644 --- a/app/admin/routes.py +++ b/app/admin/routes.py @@ -18,7 +18,7 @@ from app.community.util import save_icon_file, save_banner_file from app.models import AllowedInstances, BannedInstances, ActivityPubLog, utcnow, Site, Community, CommunityMember, \ User, Instance, File, Report, Topic, UserRegistration, Role, Post from app.utils import render_template, permission_required, set_setting, get_setting, gibberish, markdown_to_html, \ - moderating_communities, joined_communities, finalize_user_setup, theme_list, blocked_phrases + moderating_communities, joined_communities, finalize_user_setup, theme_list, blocked_phrases, blocked_referrers from app.admin import bp @@ -80,12 +80,14 @@ def admin_misc(): site.reports_email_admins = form.reports_email_admins.data site.registration_mode = form.registration_mode.data site.application_question = form.application_question.data + site.auto_decline_referrers = form.auto_decline_referrers.data site.log_activitypub_json = form.log_activitypub_json.data site.updated = utcnow() site.default_theme = form.default_theme.data if site.id is None: db.session.add(site) db.session.commit() + cache.delete_memoized(blocked_referrers) flash('Settings saved.') elif request.method == 'GET': form.enable_downvotes.data = site.enable_downvotes @@ -97,6 +99,7 @@ def admin_misc(): form.reports_email_admins.data = site.reports_email_admins form.registration_mode.data = site.registration_mode form.application_question.data = site.application_question + form.auto_decline_referrers.data = site.auto_decline_referrers form.log_activitypub_json.data = site.log_activitypub_json form.default_theme.data = site.default_theme if site.default_theme is not None else '' return render_template('admin/misc.html', title=_('Misc settings'), form=form, diff --git a/app/auth/routes.py b/app/auth/routes.py index 14f232be..4117c02c 100644 --- a/app/auth/routes.py +++ b/app/auth/routes.py @@ -12,7 +12,7 @@ from app.auth.util import random_token, normalize_utf from app.email import send_verification_email, send_password_reset_email from app.models import User, utcnow, IpBan, UserRegistration, Notification, Site from app.utils import render_template, ip_address, user_ip_banned, user_cookie_banned, banned_ip_addresses, \ - finalize_user_setup + finalize_user_setup, blocked_referrers @bp.route('/login', methods=['GET', 'POST']) @@ -98,6 +98,11 @@ def register(): if form.user_name.data in disallowed_usernames: flash(_('Sorry, you cannot use that user name'), 'error') else: + for referrer in blocked_referrers(): + if referrer in session.get('Referer'): + resp = make_response(redirect(url_for('auth.please_wait'))) + resp.set_cookie('sesion', '17489047567495', expires=datetime(year=2099, month=12, day=30)) + return resp verification_token = random_token(16) form.user_name.data = form.user_name.data.strip() before_normalize = form.user_name.data diff --git a/app/models.py b/app/models.py index 5c0c4eb3..0ae48c51 100644 --- a/app/models.py +++ b/app/models.py @@ -1171,7 +1171,7 @@ class Site(db.Model): allowlist = db.Column(db.Text, default='') blocklist = db.Column(db.Text, default='') blocked_phrases = db.Column(db.Text, default='') # discard incoming content with these phrases - auto_decline_referrers = db.Column(db.Text, default='rdrama.net') # automatically decline registration requests if the referrer is one of these + auto_decline_referrers = db.Column(db.Text, default='rdrama.net\nahrefs.com') # automatically decline registration requests if the referrer is one of these created_at = db.Column(db.DateTime, default=utcnow) updated = db.Column(db.DateTime, default=utcnow) last_active = db.Column(db.DateTime, default=utcnow) diff --git a/app/utils.py b/app/utils.py index aef7ac30..5c6d2fcd 100644 --- a/app/utils.py +++ b/app/utils.py @@ -339,6 +339,14 @@ def blocked_phrases() -> List[str]: return [] +@cache.memoize(timeout=86400) +def blocked_referrers() -> List[str]: + site = Site.query.get(1) + if site.auto_decline_referrers: + return [referrer for referrer in site.auto_decline_referrers.split('\n') if referrer != ''] + else: + return [] + def retrieve_block_list(): try: response = requests.get('https://raw.githubusercontent.com/rimu/no-qanon/master/domains.txt', timeout=1) From ab60bc753b3bab4985e42e3b454eccb6c1246bf8 Mon Sep 17 00:00:00 2001 From: rimu <3310831+rimu@users.noreply.github.com> Date: Fri, 22 Mar 2024 20:49:35 +1300 Subject: [PATCH 15/21] correct Accept header --- app/community/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/community/util.py b/app/community/util.py index 758f0bd3..44642fb0 100644 --- a/app/community/util.py +++ b/app/community/util.py @@ -122,7 +122,7 @@ def retrieve_mods_and_backfill(community_id: int): c.last_active = Post.query.filter(Post.community_id == community_id).order_by(desc(Post.posted_at)).first().posted_at db.session.commit() if community.ap_featured_url: - featured_request = get_request(community.ap_featured_url, headers={'Accept': 'application/activityjson'}) + featured_request = get_request(community.ap_featured_url, headers={'Accept': 'application/activity+json'}) if featured_request.status_code == 200: featured_data = featured_request.json() featured_request.close() From 2e2406c0d6348bc11f67f1148045e1f0eff286ae Mon Sep 17 00:00:00 2001 From: rimu <3310831+rimu@users.noreply.github.com> Date: Sat, 23 Mar 2024 06:52:55 +1300 Subject: [PATCH 16/21] cloudflare cache flush wip --- app/main/routes.py | 18 +++++++++++++++++- config.py | 3 +++ env.sample | 3 +++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/app/main/routes.py b/app/main/routes.py index 44cca30e..3b259008 100644 --- a/app/main/routes.py +++ b/app/main/routes.py @@ -5,6 +5,7 @@ from random import randint import flask import markdown2 +import requests from sqlalchemy.sql.operators import or_, and_ from app import db, cache @@ -276,7 +277,22 @@ def list_files(directory): @bp.route('/test') def test(): - x = find_actor_or_create('artporn@lemm.ee') + headers = { + 'Authorization': f"Bearer {current_app.config['CLOUDFLARE_API_TOKEN']}", + 'Content-Type': 'application/json' + } + body = { + 'files': [''] + } + zone_id = current_app.config['CLOUDFLARE_ZONE_ID'] + response = requests.request( + 'POST', + f'https://api.cloudflare.com/client/v4/zones/{zone_id}/purge_cache', + headers=headers, + data=body, + timeout=5, + ) + return 'ok' users_to_notify = User.query.join(Notification, User.id == Notification.user_id).filter( diff --git a/config.py b/config.py index f26b3fe4..cb39d944 100644 --- a/config.py +++ b/config.py @@ -46,3 +46,6 @@ class Config(object): SESSION_COOKIE_SECURE = True SESSION_COOKIE_HTTPONLY = True SESSION_COOKIE_SAMESITE = 'Lax' + + CLOUDFLARE_API_TOKEN = os.environ.get('CLOUDFLARE_API_TOKEN') or '' + CLOUDFLARE_ZONE_ID = os.environ.get('CLOUDFLARE_ZONE_ID') or '' diff --git a/env.sample b/env.sample index c03c483b..e7d44a85 100644 --- a/env.sample +++ b/env.sample @@ -26,3 +26,6 @@ FLASK_APP = 'pyfedi.py' SENTRY_DSN = '' AWS_REGION = 'ap-southeast-2' + +CLOUDFLARE_API_TOKEN = '' +CLOUDFLARE_ZONE_ID = '' \ No newline at end of file From c374f65a27ea268bd0f904efdf0027d86108d7d4 Mon Sep 17 00:00:00 2001 From: rimu <3310831+rimu@users.noreply.github.com> Date: Sat, 23 Mar 2024 07:08:28 +1300 Subject: [PATCH 17/21] limit admin user editing power #116 --- app/admin/forms.py | 14 ++------ app/admin/routes.py | 56 ++++++------------------------ app/templates/admin/edit_user.html | 23 ++++++------ 3 files changed, 22 insertions(+), 71 deletions(-) diff --git a/app/admin/forms.py b/app/admin/forms.py index c47280f9..43c48e51 100644 --- a/app/admin/forms.py +++ b/app/admin/forms.py @@ -167,26 +167,16 @@ class AddUserForm(FlaskForm): class EditUserForm(FlaskForm): - about = TextAreaField(_l('Bio'), validators=[Optional(), Length(min=3, max=5000)]) - email = StringField(_l('Email address'), validators=[Optional(), Length(max=255)]) - matrix_user_id = StringField(_l('Matrix User ID'), validators=[Optional(), Length(max=255)]) - profile_file = FileField(_l('Avatar image')) - banner_file = FileField(_l('Top banner image')) bot = BooleanField(_l('This profile is a bot')) verified = BooleanField(_l('Email address is verified')) banned = BooleanField(_l('Banned')) - newsletter = BooleanField(_l('Subscribe to email newsletter')) - ignore_bots = BooleanField(_l('Hide posts by bots')) - nsfw = BooleanField(_l('Show NSFW posts')) - nsfl = BooleanField(_l('Show NSFL posts')) - searchable = BooleanField(_l('Show profile in user list')) - indexable = BooleanField(_l('Allow search engines to index this profile')) - manually_approves_followers = BooleanField(_l('Manually approve followers')) role_options = [(2, _l('User')), (3, _l('Staff')), (4, _l('Admin')), ] role = SelectField(_l('Role'), choices=role_options, default=2, coerce=int) + remove_avatar = BooleanField(_l('Remove avatar')) + remove_banner = BooleanField(_l('Remove banner')) submit = SubmitField(_l('Save')) diff --git a/app/admin/routes.py b/app/admin/routes.py index 776d1169..2ecc2c54 100644 --- a/app/admin/routes.py +++ b/app/admin/routes.py @@ -535,46 +535,20 @@ def admin_user_edit(user_id): form = EditUserForm() user = User.query.get_or_404(user_id) if form.validate_on_submit(): - user.about = form.about.data - user.email = form.email.data - user.about_html = markdown_to_html(form.about.data) - user.matrix_user_id = form.matrix_user_id.data user.bot = form.bot.data user.verified = form.verified.data user.banned = form.banned.data - profile_file = request.files['profile_file'] - if profile_file and profile_file.filename != '': - # remove old avatar - if user.avatar_id: - file = File.query.get(user.avatar_id) - file.delete_from_disk() - user.avatar_id = None - db.session.delete(file) + if form.remove_avatar.data and user.avatar_id: + file = File.query.get(user.avatar_id) + file.delete_from_disk() + user.avatar_id = None + db.session.delete(file) - # add new avatar - file = save_icon_file(profile_file, 'users') - if file: - user.avatar = file - banner_file = request.files['banner_file'] - if banner_file and banner_file.filename != '': - # remove old cover - if user.cover_id: - file = File.query.get(user.cover_id) - file.delete_from_disk() - user.cover_id = None - db.session.delete(file) - - # add new cover - file = save_banner_file(banner_file, 'users') - if file: - user.cover = file - user.newsletter = form.newsletter.data - user.ignore_bots = form.ignore_bots.data - user.show_nsfw = form.nsfw.data - user.show_nsfl = form.nsfl.data - user.searchable = form.searchable.data - user.indexable = form.indexable.data - user.ap_manually_approves_followers = form.manually_approves_followers.data + if form.remove_banner.data and user.cover_id: + file = File.query.get(user.cover_id) + file.delete_from_disk() + user.cover_id = None + db.session.delete(file) # Update user roles. The UI only lets the user choose 1 role but the DB structure allows for multiple roles per user. db.session.execute(text('DELETE FROM user_role WHERE user_id = :user_id'), {'user_id': user.id}) @@ -589,19 +563,9 @@ def admin_user_edit(user_id): else: if not user.is_local(): flash(_('This is a remote user - most settings here will be regularly overwritten with data from the original server.'), 'warning') - form.about.data = user.about - form.email.data = user.email - form.matrix_user_id.data = user.matrix_user_id - form.newsletter.data = user.newsletter form.bot.data = user.bot form.verified.data = user.verified form.banned.data = user.banned - form.ignore_bots.data = user.ignore_bots - form.nsfw.data = user.show_nsfw - form.nsfl.data = user.show_nsfl - form.searchable.data = user.searchable - form.indexable.data = user.indexable - form.manually_approves_followers.data = user.ap_manually_approves_followers if user.roles and user.roles.count() > 0: form.role.data = user.roles[0].id diff --git a/app/templates/admin/edit_user.html b/app/templates/admin/edit_user.html index 9e2d7b8c..1bf6b475 100644 --- a/app/templates/admin/edit_user.html +++ b/app/templates/admin/edit_user.html @@ -17,29 +17,26 @@

{{ _('Edit %(user_name)s (%(display_name)s)', user_name=user.user_name, display_name=user.display_name()) }}

{{ form.csrf_token() }} - {{ render_field(form.about) }} - {{ render_field(form.email) }} - {{ render_field(form.matrix_user_id) }} + {{ user.about_html|safe if user.about_html }} +

Email: {{ user.email }}

+

Matrix: {{ user.matrix_user_id if user.matrix_user_id }}

{% if user.avatar_id %} {% endif %} - {{ render_field(form.profile_file) }} - Provide a square image that looks good when small. {% if user.cover_id %} {% endif %} - {{ render_field(form.banner_file) }} - Provide a wide image - letterbox orientation. {{ render_field(form.bot) }} {{ render_field(form.verified) }} {{ render_field(form.banned) }} - {{ render_field(form.newsletter) }} - {{ render_field(form.nsfw) }} - {{ render_field(form.nsfl) }} - {{ render_field(form.searchable) }} - {{ render_field(form.indexable) }} - {{ render_field(form.manually_approves_followers) }} +

receive newsletter: {{ user.newsletter }}

+

view nsfw: {{ user.nsfw }}

+

view nsfl: {{ user.nsfl }}

+

searchable: {{ user.searchable }}

+

indexable: {{ user.indexable }}

{{ render_field(form.role) }} + {{ render_field(form.remove_avatar) }} + {{ render_field(form.remove_banner) }} {{ render_field(form.submit) }}

From c18679a0f5c6db31e9b24b9ec0e1f3e36df715a9 Mon Sep 17 00:00:00 2001 From: rimu <3310831+rimu@users.noreply.github.com> Date: Sat, 23 Mar 2024 07:15:50 +1300 Subject: [PATCH 18/21] show created and last active dates #116 --- app/templates/admin/edit_user.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/templates/admin/edit_user.html b/app/templates/admin/edit_user.html index 1bf6b475..6c0885f4 100644 --- a/app/templates/admin/edit_user.html +++ b/app/templates/admin/edit_user.html @@ -18,6 +18,8 @@

{{ form.csrf_token() }} {{ user.about_html|safe if user.about_html }} +

Created: {{ moment(user.created).format('MMMM Do YYYY, h:mm:ss a') }}

+

Last active: {{ moment(user.last_seen).format('MMMM Do YYYY, h:mm:ss a') }}

Email: {{ user.email }}

Matrix: {{ user.matrix_user_id if user.matrix_user_id }}

{% if user.avatar_id %} @@ -43,8 +45,6 @@ {% if not user.is_local() %} View original profile {% endif %} - Ban - Ban + Purge

From 9287a1cbf212d523fd03c0eff6cd5e80ec434447 Mon Sep 17 00:00:00 2001 From: rimu <3310831+rimu@users.noreply.github.com> Date: Sat, 23 Mar 2024 15:12:51 +1300 Subject: [PATCH 19/21] flush cdn cache when a file is deleted --- app/main/routes.py | 18 +-------------- app/models.py | 57 ++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 56 insertions(+), 19 deletions(-) diff --git a/app/main/routes.py b/app/main/routes.py index 3b259008..68eca138 100644 --- a/app/main/routes.py +++ b/app/main/routes.py @@ -277,23 +277,7 @@ def list_files(directory): @bp.route('/test') def test(): - headers = { - 'Authorization': f"Bearer {current_app.config['CLOUDFLARE_API_TOKEN']}", - 'Content-Type': 'application/json' - } - body = { - 'files': [''] - } - zone_id = current_app.config['CLOUDFLARE_ZONE_ID'] - response = requests.request( - 'POST', - f'https://api.cloudflare.com/client/v4/zones/{zone_id}/purge_cache', - headers=headers, - data=body, - timeout=5, - ) - - return 'ok' + return '' users_to_notify = User.query.join(Notification, User.id == Notification.user_id).filter( User.ap_id == None, diff --git a/app/models.py b/app/models.py index 8d19bb59..2c999d45 100644 --- a/app/models.py +++ b/app/models.py @@ -1,7 +1,8 @@ from datetime import datetime, timedelta, date, timezone from time import time -from typing import List +from typing import List, Union +import requests from flask import current_app, escape, url_for, render_template_string from flask_login import UserMixin, current_user from sqlalchemy import or_, text @@ -11,7 +12,7 @@ from sqlalchemy.orm import backref from sqlalchemy_utils.types import TSVectorType # https://sqlalchemy-searchable.readthedocs.io/en/latest/installation.html from flask_sqlalchemy import BaseQuery from sqlalchemy_searchable import SearchQueryMixin -from app import db, login, cache +from app import db, login, cache, celery import jwt import os @@ -203,12 +204,20 @@ class File(db.Model): return f"https://{current_app.config['SERVER_NAME']}/{thumbnail_path}" def delete_from_disk(self): + purge_from_cache = [] if self.file_path and os.path.isfile(self.file_path): os.unlink(self.file_path) + purge_from_cache.append(self.file_path.replace('app/', f"https://{current_app.config['SERVER_NAME']}/")) if self.thumbnail_path and os.path.isfile(self.thumbnail_path): os.unlink(self.thumbnail_path) + purge_from_cache.append(self.thumbnail_path.replace('app/', f"https://{current_app.config['SERVER_NAME']}/")) if self.source_url and not self.source_url.startswith('http') and os.path.isfile(self.source_url): os.unlink(self.source_url) + purge_from_cache.append(self.source_url.replace('app/', f"https://{current_app.config['SERVER_NAME']}/")) + + if purge_from_cache: + flush_cdn_cache(purge_from_cache) + def filesize(self): size = 0 @@ -219,6 +228,50 @@ class File(db.Model): return size +def flush_cdn_cache(url: Union[str, List[str]]): + zone_id = current_app.config['CLOUDFLARE_ZONE_ID'] + token = current_app.config['CLOUDFLARE_API_TOKEN'] + if zone_id and token: + if current_app.debug: + flush_cdn_cache_task(url) + else: + flush_cdn_cache_task.delay(url) + + +@celery.task +def flush_cdn_cache_task(to_purge: Union[str, List[str]]): + zone_id = current_app.config['CLOUDFLARE_ZONE_ID'] + token = current_app.config['CLOUDFLARE_API_TOKEN'] + headers = { + 'Authorization': f"Bearer {token}", + 'Content-Type': 'application/json' + } + # url can be a string or a list of strings + body = '' + if isinstance(to_purge, str) and to_purge == 'all': + body = { + 'purge_everything': True + } + else: + if isinstance(to_purge, str): + body = { + 'files': [to_purge] + } + elif isinstance(to_purge, list): + body = { + 'files': to_purge + } + + if body: + response = requests.request( + 'POST', + f'https://api.cloudflare.com/client/v4/zones/{zone_id}/purge_cache', + headers=headers, + json=body, + timeout=5, + ) + + class Topic(db.Model): id = db.Column(db.Integer, primary_key=True) machine_name = db.Column(db.String(50), index=True) From eb94db5fe90a84a1fbb26eea35eff540c07c27d9 Mon Sep 17 00:00:00 2001 From: rimu <3310831+rimu@users.noreply.github.com> Date: Sat, 23 Mar 2024 15:24:34 +1300 Subject: [PATCH 20/21] add documentation about Cloudflare cache purging --- INSTALL.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/INSTALL.md b/INSTALL.md index 1b403806..37f9f4d2 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -351,6 +351,16 @@ at ~/.aws/credentials or environment variables. Details at https://boto3.amazona In your .env you need to set the AWS region you're using for SES. Something like AWS_REGION = 'ap-southeast-2'. +#### CDN + +A CDN like Cloudflare is recommended for instances with more than a handful of users. [Recommended caching settings](https://join.piefed.social/2024/02/20/how-much-difference-does-a-cdn-make-to-a-fediverse-instance/). + +PieFed has the capability to automatically remove file copies from the Cloudflare cache whenever + those files are deleted from the server. To enable this, set these variables in your .env file: + +- CLOUDFLARE_API_TOKEN - go to https://dash.cloudflare.com/profile/api-tokens and create a "Zone.Cache Purge" token. +- CLOUDFLARE_ZONE_ID - this can be found in the right hand column of your Cloudflare dashboard in the API section. + #### SMTP To use SMTP you need to set all the MAIL_* environment variables in you .env file. See env.sample for a list of them. From 1394d7baadbcbfa21c3dd474aadd92639cef0816 Mon Sep 17 00:00:00 2001 From: rimu <3310831+rimu@users.noreply.github.com> Date: Sat, 23 Mar 2024 21:23:38 +1300 Subject: [PATCH 21/21] update install docs --- INSTALL.md | 6 +++++- requirements.txt | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/INSTALL.md b/INSTALL.md index 37f9f4d2..8d016d19 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -18,7 +18,11 @@ ## Setup Database -#### Install postgresql 16 +#### Install postgresql + +PieFed should work on version 13.x or newer. If you have errors running `flask init-db`, check your postrgesql version. + +##### Install postgresql 16: For installation environments that use 'apt' as a package manager: diff --git a/requirements.txt b/requirements.txt index 4ccc950b..e547eee5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,3 +30,4 @@ redis==5.0.1 Werkzeug==2.3.3 pytesseract==0.3.10 sentry-sdk==1.40.6 +urllib3==1.26.1