Defederation subscriptions

app/admin/forms.py

@@ -47,6 +47,7 @@ class FederationForm(FlaskForm):
     allowlist = TextAreaField(_l('Allow federation with these instances'))
     use_blocklist = BooleanField(_l('Blocklist instead of allowlist'))
     blocklist = TextAreaField(_l('Deny federation with these instances'))
+    defederation_subscription = TextAreaField(_l('Auto-defederate from any instance defederated by'))
     blocked_phrases = TextAreaField(_l('Discard all posts and comments with these phrases (one per line)'))
     blocked_actors = TextAreaField(_l('Discard all posts and comments by users with these words in their name (one per line)'))
     submit = SubmitField(_l('Save'))

app/admin/routes.py

@@ -28,10 +28,11 @@ from app.constants import REPORT_STATE_NEW, REPORT_STATE_ESCALATED
 from app.email import send_welcome_email
 from app.models import AllowedInstances, BannedInstances, ActivityPubLog, utcnow, Site, Community, CommunityMember, \
     User, Instance, File, Report, Topic, UserRegistration, Role, Post, PostReply, Language, RolePermission, Domain, \
-    Tag
+    Tag, DefederationSubscription
 from app.utils import render_template, permission_required, set_setting, get_setting, gibberish, markdown_to_html, \
     moderating_communities, joined_communities, finalize_user_setup, theme_list, blocked_phrases, blocked_referrers, \
-    topic_tree, languages_for_form, menu_topics, ensure_directory_exists, add_to_modlog, get_request, file_get_contents
+    topic_tree, languages_for_form, menu_topics, ensure_directory_exists, add_to_modlog, get_request, file_get_contents, \
+    download_defeds
 from app.admin import bp
 
 
@@ -661,11 +662,23 @@ def admin_federation():
                     cache.delete_memoized(instance_allowed, allow.strip())
         if form.use_blocklist.data:
             set_setting('use_allowlist', False)
-            db.session.execute(text('DELETE FROM banned_instances'))
+            db.session.execute(text('DELETE FROM banned_instances WHERE subscription_id is null'))
             for banned in form.blocklist.data.split('\n'):
                 if banned.strip():
                     db.session.add(BannedInstances(domain=banned.strip()))
                     cache.delete_memoized(instance_blocked, banned.strip())
+
+        # update and sync defederation subscriptions
+        db.session.execute(text('DELETE FROM banned_instances WHERE subscription_id is not null'))
+        db.session.query(DefederationSubscription).delete()
+        db.session.commit()
+        for defed_subscription in form.defederation_subscription.data.split('\n'):
+            if defed_subscription.strip():
+                db.session.add(DefederationSubscription(domain=defed_subscription.strip().lower()))
+        db.session.commit()
+        for defederation_sub in DefederationSubscription.query.all():
+            download_defeds(defederation_sub.id, defederation_sub.domain)
+
         g.site.blocked_phrases = form.blocked_phrases.data
         set_setting('actor_blocked_words', form.blocked_actors.data)
         cache.delete_memoized(blocked_phrases)
@@ -678,10 +691,11 @@ def admin_federation():
     elif request.method == 'GET':
         form.use_allowlist.data = get_setting('use_allowlist', False)
         form.use_blocklist.data = not form.use_allowlist.data
-        instances = BannedInstances.query.all()
+        instances = BannedInstances.query.filter(BannedInstances.subscription_id == None).all()
         form.blocklist.data = '\n'.join([instance.domain for instance in instances])
         instances = AllowedInstances.query.all()
         form.allowlist.data = '\n'.join([instance.domain for instance in instances])
+        form.defederation_subscription.data = '\n'.join([instance.domain for instance in DefederationSubscription.query.all()])
         form.blocked_phrases.data = g.site.blocked_phrases
         form.blocked_actors.data = get_setting('actor_blocked_words', '88')
 
@@ -1550,6 +1564,8 @@ def admin_instances():
         elif filter == 'gone_forever':
             instances = instances.filter(Instance.gone_forever == True)
             title = 'Gone forever instances'
+        elif filter == 'blocked':
+            instances = instances.join(BannedInstances, BannedInstances.domain == Instance.domain)
 
     # Pagination
     instances = instances.paginate(page=page,

app/models.py

@@ -42,6 +42,7 @@ class BannedInstances(db.Model):
     reason = db.Column(db.String(256))
     initiator = db.Column(db.String(256))
     created_at = db.Column(db.DateTime, default=utcnow)
+    subscription_id = db.Column(db.Integer, db.ForeignKey('defederation_subscription.id'), index=True) # is None when the ban was done by a local admin
 
 
 class AllowedInstances(db.Model):
@@ -50,6 +51,11 @@ class AllowedInstances(db.Model):
     created_at = db.Column(db.DateTime, default=utcnow)
 
 
+class DefederationSubscription(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    domain = db.Column(db.String(256), index=True)
+
+
 class Instance(db.Model):
     id = db.Column(db.Integer, primary_key=True)
     domain = db.Column(db.String(256), index=True, unique=True)

app/templates/admin/instances.html

@@ -17,7 +17,8 @@
         <a href="{{ url_for('admin.admin_instances', filter='online') }}">{{ _('Online') }}</a> |
         <a href="{{ url_for('admin.admin_instances', filter='dormant') }}">{{ _('Dormant') }}</a> |
         <a href="{{ url_for('admin.admin_instances', filter='gone_forever') }}">{{ _('Gone forever') }}</a> |
-        <a href="{{ url_for('admin.admin_instances', filter='trusted') }}">{{ _('Trusted') }}</a>
+        <a href="{{ url_for('admin.admin_instances', filter='trusted') }}">{{ _('Trusted') }}</a> |
+        <a href="{{ url_for('admin.admin_instances', filter='blocked') }}">{{ _('Blocked') }}</a>
         <table class="table table-striped">
             <tr>
                <th>{{ _('Domain') }}</th>

app/utils.py

@@ -29,7 +29,7 @@ from sqlalchemy import text, or_
 from sqlalchemy.orm import Session
 from wtforms.fields  import SelectField, SelectMultipleField
 from wtforms.widgets import Select, html_params, ListWidget, CheckboxInput
-from app import db, cache, httpx_client
+from app import db, cache, httpx_client, celery
 import re
 from PIL import Image, ImageOps
 
@@ -1251,9 +1251,53 @@ def get_task_session() -> Session:
     return Session(bind=db.engine)
 
 
+def download_defeds(defederation_subscription_id: int, domain: str):
+    if current_app.debug:
+        download_defeds_worker(defederation_subscription_id, domain)
+    else:
+        download_defeds_worker.delay(defederation_subscription_id, domain)
+
+
+@celery.task
+def download_defeds_worker(defederation_subscription_id: int, domain: str):
+    session = get_task_session()
+    for defederation_url in retrieve_defederation_list(domain):
+        session.add(BannedInstances(domain=defederation_url, reason='auto', subscription_id=defederation_subscription_id))
+    session.commit()
+    session.close()
+
+
+def retrieve_defederation_list(domain: str) -> List[str]:
+    result = []
+    software = instance_software(domain)
+    if software == 'lemmy' or software == 'piefed':
+        try:
+            response = get_request(f'https://{domain}/api/v3/federated_instances')
+        except:
+            response = None
+        if response and response.status_code == 200:
+            instance_data = response.json()
+            for row in instance_data['federated_instances']['blocked']:
+                result.append(row['domain'])
+    else:   # Assume mastodon-compatible API
+        try:
+            response = get_request(f'https://{domain}/api/v1/instance/domain_blocks')
+        except:
+            response = None
+        if response and response.status_code == 200:
+            instance_data = response.json()
+            for row in instance_data:
+                result.append(row['domain'])
+
+    return result
+
+
+def instance_software(domain: str):
+    instance = Instance.query.filter(Instance.domain == domain).first()
+    return instance.software.lower() if instance else ''
+
+
 user2_cache = {}
-
-
 def jaccard_similarity(user1_upvoted: set, user2_id: int):
     if user2_id not in user2_cache:
         user2_upvoted_posts = ['post/' + str(id) for id in recently_upvoted_posts(user2_id)]

migrations/versions/8758c0a94f82_defederation_subscription.py

@@ -0,0 +1,48 @@
+"""defederation subscription
+
+Revision ID: 8758c0a94f82
+Revises: f961f446ae17
+Create Date: 2024-12-27 16:47:08.097093
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '8758c0a94f82'
+down_revision = 'f961f446ae17'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('defederation_subscription',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('domain', sa.String(length=256), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    with op.batch_alter_table('defederation_subscription', schema=None) as batch_op:
+        batch_op.create_index(batch_op.f('ix_defederation_subscription_domain'), ['domain'], unique=False)
+
+    with op.batch_alter_table('banned_instances', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('subscription_id', sa.Integer(), nullable=True))
+        batch_op.create_index(batch_op.f('ix_banned_instances_subscription_id'), ['subscription_id'], unique=False)
+        batch_op.create_foreign_key(None, 'defederation_subscription', ['subscription_id'], ['id'])
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('banned_instances', schema=None) as batch_op:
+        batch_op.drop_constraint(None, type_='foreignkey')
+        batch_op.drop_index(batch_op.f('ix_banned_instances_subscription_id'))
+        batch_op.drop_column('subscription_id')
+
+    with op.batch_alter_table('defederation_subscription', schema=None) as batch_op:
+        batch_op.drop_index(batch_op.f('ix_defederation_subscription_domain'))
+
+    op.drop_table('defederation_subscription')
+    # ### end Alembic commands ###