From de11aa6c50cb1f5d96a8560e702543284e7ba26a Mon Sep 17 00:00:00 2001 From: rimu <3310831+rimu@users.noreply.github.com> Date: Fri, 27 Dec 2024 20:20:16 +1300 Subject: [PATCH] Defederation subscriptions --- app/admin/forms.py | 1 + app/admin/routes.py | 24 +++++++-- app/models.py | 6 +++ app/templates/admin/instances.html | 3 +- app/utils.py | 50 +++++++++++++++++-- .../8758c0a94f82_defederation_subscription.py | 48 ++++++++++++++++++ 6 files changed, 124 insertions(+), 8 deletions(-) create mode 100644 migrations/versions/8758c0a94f82_defederation_subscription.py diff --git a/app/admin/forms.py b/app/admin/forms.py index 59597936..dc79d85f 100644 --- a/app/admin/forms.py +++ b/app/admin/forms.py @@ -47,6 +47,7 @@ class FederationForm(FlaskForm): allowlist = TextAreaField(_l('Allow federation with these instances')) use_blocklist = BooleanField(_l('Blocklist instead of allowlist')) blocklist = TextAreaField(_l('Deny federation with these instances')) + defederation_subscription = TextAreaField(_l('Auto-defederate from any instance defederated by')) blocked_phrases = TextAreaField(_l('Discard all posts and comments with these phrases (one per line)')) blocked_actors = TextAreaField(_l('Discard all posts and comments by users with these words in their name (one per line)')) submit = SubmitField(_l('Save')) diff --git a/app/admin/routes.py b/app/admin/routes.py index 579dc19d..e4978f38 100644 --- a/app/admin/routes.py +++ b/app/admin/routes.py @@ -28,10 +28,11 @@ from app.constants import REPORT_STATE_NEW, REPORT_STATE_ESCALATED from app.email import send_welcome_email from app.models import AllowedInstances, BannedInstances, ActivityPubLog, utcnow, Site, Community, CommunityMember, \ User, Instance, File, Report, Topic, UserRegistration, Role, Post, PostReply, Language, RolePermission, Domain, \ - Tag + Tag, DefederationSubscription from app.utils import render_template, permission_required, set_setting, get_setting, gibberish, markdown_to_html, \ moderating_communities, joined_communities, finalize_user_setup, theme_list, blocked_phrases, blocked_referrers, \ - topic_tree, languages_for_form, menu_topics, ensure_directory_exists, add_to_modlog, get_request, file_get_contents + topic_tree, languages_for_form, menu_topics, ensure_directory_exists, add_to_modlog, get_request, file_get_contents, \ + download_defeds from app.admin import bp @@ -661,11 +662,23 @@ def admin_federation(): cache.delete_memoized(instance_allowed, allow.strip()) if form.use_blocklist.data: set_setting('use_allowlist', False) - db.session.execute(text('DELETE FROM banned_instances')) + db.session.execute(text('DELETE FROM banned_instances WHERE subscription_id is null')) for banned in form.blocklist.data.split('\n'): if banned.strip(): db.session.add(BannedInstances(domain=banned.strip())) cache.delete_memoized(instance_blocked, banned.strip()) + + # update and sync defederation subscriptions + db.session.execute(text('DELETE FROM banned_instances WHERE subscription_id is not null')) + db.session.query(DefederationSubscription).delete() + db.session.commit() + for defed_subscription in form.defederation_subscription.data.split('\n'): + if defed_subscription.strip(): + db.session.add(DefederationSubscription(domain=defed_subscription.strip().lower())) + db.session.commit() + for defederation_sub in DefederationSubscription.query.all(): + download_defeds(defederation_sub.id, defederation_sub.domain) + g.site.blocked_phrases = form.blocked_phrases.data set_setting('actor_blocked_words', form.blocked_actors.data) cache.delete_memoized(blocked_phrases) @@ -678,10 +691,11 @@ def admin_federation(): elif request.method == 'GET': form.use_allowlist.data = get_setting('use_allowlist', False) form.use_blocklist.data = not form.use_allowlist.data - instances = BannedInstances.query.all() + instances = BannedInstances.query.filter(BannedInstances.subscription_id == None).all() form.blocklist.data = '\n'.join([instance.domain for instance in instances]) instances = AllowedInstances.query.all() form.allowlist.data = '\n'.join([instance.domain for instance in instances]) + form.defederation_subscription.data = '\n'.join([instance.domain for instance in DefederationSubscription.query.all()]) form.blocked_phrases.data = g.site.blocked_phrases form.blocked_actors.data = get_setting('actor_blocked_words', '88') @@ -1550,6 +1564,8 @@ def admin_instances(): elif filter == 'gone_forever': instances = instances.filter(Instance.gone_forever == True) title = 'Gone forever instances' + elif filter == 'blocked': + instances = instances.join(BannedInstances, BannedInstances.domain == Instance.domain) # Pagination instances = instances.paginate(page=page, diff --git a/app/models.py b/app/models.py index 457d75b7..9944e897 100644 --- a/app/models.py +++ b/app/models.py @@ -42,6 +42,7 @@ class BannedInstances(db.Model): reason = db.Column(db.String(256)) initiator = db.Column(db.String(256)) created_at = db.Column(db.DateTime, default=utcnow) + subscription_id = db.Column(db.Integer, db.ForeignKey('defederation_subscription.id'), index=True) # is None when the ban was done by a local admin class AllowedInstances(db.Model): @@ -50,6 +51,11 @@ class AllowedInstances(db.Model): created_at = db.Column(db.DateTime, default=utcnow) +class DefederationSubscription(db.Model): + id = db.Column(db.Integer, primary_key=True) + domain = db.Column(db.String(256), index=True) + + class Instance(db.Model): id = db.Column(db.Integer, primary_key=True) domain = db.Column(db.String(256), index=True, unique=True) diff --git a/app/templates/admin/instances.html b/app/templates/admin/instances.html index a97343c6..d6e841a8 100644 --- a/app/templates/admin/instances.html +++ b/app/templates/admin/instances.html @@ -17,7 +17,8 @@ {{ _('Online') }} | {{ _('Dormant') }} | {{ _('Gone forever') }} | - {{ _('Trusted') }} + {{ _('Trusted') }} | + {{ _('Blocked') }} diff --git a/app/utils.py b/app/utils.py index c08c0a22..ae5299be 100644 --- a/app/utils.py +++ b/app/utils.py @@ -29,7 +29,7 @@ from sqlalchemy import text, or_ from sqlalchemy.orm import Session from wtforms.fields import SelectField, SelectMultipleField from wtforms.widgets import Select, html_params, ListWidget, CheckboxInput -from app import db, cache, httpx_client +from app import db, cache, httpx_client, celery import re from PIL import Image, ImageOps @@ -1251,9 +1251,53 @@ def get_task_session() -> Session: return Session(bind=db.engine) +def download_defeds(defederation_subscription_id: int, domain: str): + if current_app.debug: + download_defeds_worker(defederation_subscription_id, domain) + else: + download_defeds_worker.delay(defederation_subscription_id, domain) + + +@celery.task +def download_defeds_worker(defederation_subscription_id: int, domain: str): + session = get_task_session() + for defederation_url in retrieve_defederation_list(domain): + session.add(BannedInstances(domain=defederation_url, reason='auto', subscription_id=defederation_subscription_id)) + session.commit() + session.close() + + +def retrieve_defederation_list(domain: str) -> List[str]: + result = [] + software = instance_software(domain) + if software == 'lemmy' or software == 'piefed': + try: + response = get_request(f'https://{domain}/api/v3/federated_instances') + except: + response = None + if response and response.status_code == 200: + instance_data = response.json() + for row in instance_data['federated_instances']['blocked']: + result.append(row['domain']) + else: # Assume mastodon-compatible API + try: + response = get_request(f'https://{domain}/api/v1/instance/domain_blocks') + except: + response = None + if response and response.status_code == 200: + instance_data = response.json() + for row in instance_data: + result.append(row['domain']) + + return result + + +def instance_software(domain: str): + instance = Instance.query.filter(Instance.domain == domain).first() + return instance.software.lower() if instance else '' + + user2_cache = {} - - def jaccard_similarity(user1_upvoted: set, user2_id: int): if user2_id not in user2_cache: user2_upvoted_posts = ['post/' + str(id) for id in recently_upvoted_posts(user2_id)] diff --git a/migrations/versions/8758c0a94f82_defederation_subscription.py b/migrations/versions/8758c0a94f82_defederation_subscription.py new file mode 100644 index 00000000..149648e9 --- /dev/null +++ b/migrations/versions/8758c0a94f82_defederation_subscription.py @@ -0,0 +1,48 @@ +"""defederation subscription + +Revision ID: 8758c0a94f82 +Revises: f961f446ae17 +Create Date: 2024-12-27 16:47:08.097093 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '8758c0a94f82' +down_revision = 'f961f446ae17' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('defederation_subscription', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('domain', sa.String(length=256), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + with op.batch_alter_table('defederation_subscription', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_defederation_subscription_domain'), ['domain'], unique=False) + + with op.batch_alter_table('banned_instances', schema=None) as batch_op: + batch_op.add_column(sa.Column('subscription_id', sa.Integer(), nullable=True)) + batch_op.create_index(batch_op.f('ix_banned_instances_subscription_id'), ['subscription_id'], unique=False) + batch_op.create_foreign_key(None, 'defederation_subscription', ['subscription_id'], ['id']) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('banned_instances', schema=None) as batch_op: + batch_op.drop_constraint(None, type_='foreignkey') + batch_op.drop_index(batch_op.f('ix_banned_instances_subscription_id')) + batch_op.drop_column('subscription_id') + + with op.batch_alter_table('defederation_subscription', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_defederation_subscription_domain')) + + op.drop_table('defederation_subscription') + # ### end Alembic commands ###
{{ _('Domain') }}