fake news domain blocking and notifying

app/activitypub/routes.py

@@ -8,7 +8,7 @@ from app.post.routes import continue_discussion, show_post
 from app.user.routes import show_profile
 from app.constants import POST_TYPE_LINK, POST_TYPE_IMAGE, SUBSCRIPTION_MEMBER
 from app.models import User, Community, CommunityJoinRequest, CommunityMember, CommunityBan, ActivityPubLog, Post, \
-    PostReply, Instance, PostVote, PostReplyVote, File, AllowedInstances, BannedInstances, utcnow, Site
+    PostReply, Instance, PostVote, PostReplyVote, File, AllowedInstances, BannedInstances, utcnow, Site, Notification
 from app.activitypub.util import public_key, users_total, active_half_year, active_month, local_posts, local_comments, \
     post_to_activity, find_actor_or_create, default_context, instance_blocked, find_reply_parent, find_liked_object, \
     lemmy_site_data, instance_weight, is_activitypub_request, downvote_post_reply, downvote_post, upvote_post_reply, \
@@ -18,8 +18,6 @@ from app.utils import gibberish, get_setting, is_image_url, allowlist_html, html
     domain_from_url, markdown_to_html, community_membership, ap_datetime, markdown_to_text
 import werkzeug.exceptions
 
-INBOX = []
-
 
 @bp.route('/.well-known/webfinger')
 def webfinger():
@@ -402,11 +400,29 @@ def process_inbox_request(request_json, activitypublog_id):
                                         else:
                                             post.type = POST_TYPE_LINK
                                         domain = domain_from_url(post.url)
-                                        if not domain.banned:
+                                        # notify about links to banned websites.
-                                            post.domain_id = domain.id
+                                        already_notified = set()  # often admins and mods are the same people - avoid notifying them twice
-                                        else:
+                                        if domain.notify_mods:
+                                            for community_member in post.community.moderators():
+                                                notify = Notification(title='Suspicious content', url=post.ap_id,
+                                                                      user_id=community_member.user_id,
+                                                                      author_id=user.id)
+                                                db.session.add(notify)
+                                                already_notified.add(community_member.user_id)
+                                        if domain.notify_admins:
+                                            for admin in Site.admins():
+                                                if admin.id not in already_notified:
+                                                    notify = Notification(title='Suspicious content',
+                                                                          url=post.ap_id, user_id=admin.id,
+                                                                          author_id=user.id)
+                                                    db.session.add(notify)
+                                        if domain.banned:
                                             post = None
                                             activity_log.exception_message = domain.name + ' is blocked by admin'
+                                        if not domain.banned:
+                                            domain.post_count += 1
+                                            post.domain = domain
+
                                 if 'image' in request_json['object']:
                                     image = File(source_url=request_json['object']['image']['url'])
                                     db.session.add(image)
@@ -516,23 +532,41 @@ def process_inbox_request(request_json, activitypublog_id):
                                             else:
                                                 post.type = POST_TYPE_LINK
                                             domain = domain_from_url(post.url)
-                                            if not domain.banned:
+                                            # notify about links to banned websites.
-                                                post.domain_id = domain.id
+                                            already_notified = set()  # often admins and mods are the same people - avoid notifying them twice
-                                            else:
+                                            if domain.notify_mods:
+                                                for community_member in post.community.moderators():
+                                                    notify = Notification(title='Suspicious content', url=post.ap_id,
+                                                                          user_id=community_member.user_id,
+                                                                          author_id=user.id)
+                                                    db.session.add(notify)
+                                                    already_notified.add(community_member.user_id)
+                                            if domain.notify_admins:
+                                                for admin in Site.admins():
+                                                    if admin.id not in already_notified:
+                                                        notify = Notification(title='Suspicious content',
+                                                                              url=post.ap_id, user_id=admin.id,
+                                                                              author_id=user.id)
+                                                        db.session.add(notify)
+                                            if domain.banned:
                                                 post = None
                                                 activity_log.exception_message = domain.name + ' is blocked by admin'
-                                    if 'image' in request_json['object']['object']:
+                                            if not domain.banned:
+                                                domain.post_count += 1
+                                                post.domain = domain
+
+                                    if 'image' in request_json['object']['object'] and post:
                                         image = File(source_url=request_json['object']['object']['image']['url'])
                                         db.session.add(image)
                                         post.image = image
 
-                                    if post is not None:
+                                        if post is not None:
-                                        db.session.add(post)
+                                            db.session.add(post)
-                                        community.post_count += 1
+                                            community.post_count += 1
-                                        activity_log.result = 'success'
+                                            activity_log.result = 'success'
-                                        db.session.commit()
+                                            db.session.commit()
-                                        if post.image_id:
+                                            if post.image_id:
-                                            make_image_sizes(post.image_id, 266, None, 'posts')
+                                                make_image_sizes(post.image_id, 266, None, 'posts')
                                 else:
                                     post_id, parent_comment_id, root_id = find_reply_parent(in_reply_to)
                                     if post_id or parent_comment_id or root_id:
@@ -924,6 +958,8 @@ def process_inbox_request(request_json, activitypublog_id):
                 # Flush the caches of any major object that was created. To be sure.
                 if 'user' in vars() and user is not None:
                     user.flush_cache()
+                    if user.instance_id:
+                        user.instance.last_seen = utcnow()
                 # if 'community' in vars() and community is not None:
                 #    community.flush_cache()
                 if 'post' in vars() and post is not None:
@@ -1095,7 +1131,7 @@ def comment_ap(comment_id):
         return resp
     else:
         reply = PostReply.query.get(comment_id)
-        continue_discussion(reply.post.id, comment_id)
+        return continue_discussion(reply.post.id, comment_id)
 
 
 @bp.route('/post/<int:post_id>', methods=['GET', 'POST'])

app/activitypub/util.py

@@ -9,7 +9,7 @@ from flask import current_app, request, g
 from sqlalchemy import text
 from app import db, cache, constants, celery
 from app.models import User, Post, Community, BannedInstances, File, PostReply, AllowedInstances, Instance, utcnow, \
-    PostVote, PostReplyVote, ActivityPubLog
+    PostVote, PostReplyVote, ActivityPubLog, Notification, Site
 import time
 import base64
 import requests
@@ -439,12 +439,26 @@ def post_json_to_model(post_json, user, community) -> Post:
                 post.type = POST_TYPE_IMAGE
             else:
                 post.type = POST_TYPE_LINK
+
             domain = domain_from_url(post.url)
-            if not domain.banned:
+            # notify about links to banned websites.
-                post.domain_id = domain.id
+            already_notified = set()        # often admins and mods are the same people - avoid notifying them twice
-            else:
+            if domain.notify_mods:
+                for community_member in post.community.moderators():
+                    notify = Notification(title='Suspicious content', url=post.ap_id, user_id=community_member.user_id, author_id=user.id)
+                    db.session.add(notify)
+                    already_notified.add(community_member.user_id)
+            if domain.notify_admins:
+                for admin in Site.admins():
+                    if admin.id not in already_notified:
+                        notify = Notification(title='Suspicious content', url=post.ap_id, user_id=admin.id, author_id=user.id)
+                        db.session.add(notify)
+            if domain.banned:
                 post = None
-    if 'image' in post_json:
+            if not domain.banned:
+                domain.post_count += 1
+                post.domain = domain
+    if 'image' in post_json and post:
         image = File(source_url=post_json['image']['url'])
         db.session.add(image)
         post.image = image
@@ -620,7 +634,7 @@ def find_instance_id(server):
         db.session.add(new_instance)
         db.session.commit()
 
-        # Spawn background task
+        # Spawn background task to fill in more details
         refresh_instance_profile(new_instance.id)
 
         return new_instance.id

app/auth/routes.py

@@ -76,7 +76,7 @@ def register():
                 verification_token = random_token(16)
                 form.user_name.data = form.user_name.data.strip()
                 user = User(user_name=form.user_name.data, email=form.real_email.data,
-                            verification_token=verification_token)
+                            verification_token=verification_token, instance=1)
                 user.set_password(form.password.data)
                 db.session.add(user)
                 db.session.commit()

app/models.py

@@ -264,6 +264,7 @@ class User(UserMixin, db.Model):
 
     avatar = db.relationship('File', lazy='joined', foreign_keys=[avatar_id], single_parent=True, cascade="all, delete-orphan")
     cover = db.relationship('File', lazy='joined', foreign_keys=[cover_id], single_parent=True, cascade="all, delete-orphan")
+    instance = db.relationship('Instance', lazy='joined', foreign_keys=[instance_id])
 
     ap_id = db.Column(db.String(255), index=True)           # e.g. username@server
     ap_profile_id = db.Column(db.String(255), index=True)   # e.g. https://server/u/username
@@ -665,6 +666,8 @@ class Domain(db.Model):
     name = db.Column(db.String(255), index=True)
     post_count = db.Column(db.Integer, default=0)
     banned = db.Column(db.Boolean, default=False, index=True) # Domains can be banned site-wide (by admin) or DomainBlock'ed by users
+    notify_mods = db.Column(db.Boolean, default=False, index=True)
+    notify_admins = db.Column(db.Boolean, default=False, index=True)
 
 
 class DomainBlock(db.Model):
@@ -738,14 +741,20 @@ class Instance(db.Model):
     version = db.Column(db.String(50))
     created_at = db.Column(db.DateTime, default=utcnow)
     updated_at = db.Column(db.DateTime, default=utcnow)
+    last_seen = db.Column(db.DateTime, default=utcnow)      # When an Activity was received from them
+    last_successful_send = db.Column(db.DateTime)           # When we successfully sent them an Activity
+    failures = db.Column(db.Integer, default=0)             # How many times we failed to send (reset to 0 after every successful send)
+    most_recent_attempt = db.Column(db.DateTime)            # When the most recent failure was
+    dormant = db.Column(db.Boolean, default=False)          # True once this instance is considered offline and not worth sending to any more
+    start_trying_again = db.Column(db.DateTime)             # When to start trying again. Should grow exponentially with each failure.
+    gone_forever = db.Column(db.Boolean, default=False)     # True once this instance is considered offline forever - never start trying again
 
     posts = db.relationship('Post', backref='instance', lazy='dynamic')
     post_replies = db.relationship('PostReply', backref='instance', lazy='dynamic')
     communities = db.relationship('Community', backref='instance', lazy='dynamic')
 
-    def alive(self):
+    def online(self):
-        # todo: determine aliveness based on number of failed connection attempts, etc
+        return not self.dormant and not self.gone_forever
-        return True
 
 
 class InstanceBlock(db.Model):
@@ -841,8 +850,8 @@ class Notification(db.Model):
     title = db.Column(db.String(50))
     url = db.Column(db.String(512))
     read = db.Column(db.Boolean, default=False)
-    user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
+    user_id = db.Column(db.Integer, db.ForeignKey('user.id'))       # who the notification should go to
-    author_id = db.Column(db.Integer, db.ForeignKey('user.id'))
+    author_id = db.Column(db.Integer, db.ForeignKey('user.id'))     # the person who caused the notification to happen
     created_at = db.Column(db.DateTime, default=utcnow)
 
 
@@ -886,6 +895,10 @@ class Site(db.Model):
     updated = db.Column(db.DateTime, default=utcnow)
     last_active = db.Column(db.DateTime, default=utcnow)
 
+    @staticmethod
+    def admins() -> List[User]:
+        return User.query.filter_by(deleted=False, banned=False).join(user_role).filter(user_role.c.role_id == 4).all()
+
 
 @login.user_loader
 def load_user(id):

app/user/routes.py

@@ -261,7 +261,7 @@ def send_deletion_requests(user_id):
             "type": "Delete"
         }
         for instance in instances:
-            if instance.inbox and instance.alive() and instance.id != 1: # instance id 1 is always the current instance
+            if instance.inbox and instance.online() and instance.id != 1: # instance id 1 is always the current instance
                 post_request(instance.inbox, payload, user.private_key, f"{user.profile_id()}#main-key")
 
         sleep(5)

migrations/versions/b58c4301d1ad_link_sharing_monitoring.py

@@ -0,0 +1,38 @@
+"""link sharing monitoring
+
+Revision ID: b58c4301d1ad
+Revises: ea5650ac4628
+Create Date: 2023-12-30 08:08:57.954434
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'b58c4301d1ad'
+down_revision = 'ea5650ac4628'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('domain', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('notify_mods', sa.Boolean(), nullable=True))
+        batch_op.add_column(sa.Column('notify_admins', sa.Boolean(), nullable=True))
+        batch_op.create_index(batch_op.f('ix_domain_notify_admins'), ['notify_admins'], unique=False)
+        batch_op.create_index(batch_op.f('ix_domain_notify_mods'), ['notify_mods'], unique=False)
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('domain', schema=None) as batch_op:
+        batch_op.drop_index(batch_op.f('ix_domain_notify_mods'))
+        batch_op.drop_index(batch_op.f('ix_domain_notify_admins'))
+        batch_op.drop_column('notify_admins')
+        batch_op.drop_column('notify_mods')
+
+    # ### end Alembic commands ###

migrations/versions/ea5650ac4628_instance_health.py

@@ -0,0 +1,44 @@
+"""instance health
+
+Revision ID: ea5650ac4628
+Revises: 88d210da7f2b
+Create Date: 2023-12-29 20:26:42.527252
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'ea5650ac4628'
+down_revision = '88d210da7f2b'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('instance', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('last_seen', sa.DateTime(), nullable=True))
+        batch_op.add_column(sa.Column('last_successful_send', sa.DateTime(), nullable=True))
+        batch_op.add_column(sa.Column('failures', sa.Integer(), nullable=True))
+        batch_op.add_column(sa.Column('most_recent_attempt', sa.DateTime(), nullable=True))
+        batch_op.add_column(sa.Column('dormant', sa.Boolean(), nullable=True))
+        batch_op.add_column(sa.Column('start_trying_again', sa.DateTime(), nullable=True))
+        batch_op.add_column(sa.Column('gone_forever', sa.Boolean(), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('instance', schema=None) as batch_op:
+        batch_op.drop_column('gone_forever')
+        batch_op.drop_column('start_trying_again')
+        batch_op.drop_column('dormant')
+        batch_op.drop_column('most_recent_attempt')
+        batch_op.drop_column('failures')
+        batch_op.drop_column('last_successful_send')
+        batch_op.drop_column('last_seen')
+
+    # ### end Alembic commands ###