diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index ab1aa1e2..99b86ed3 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -8,7 +8,7 @@ from app.post.routes import continue_discussion, show_post from app.user.routes import show_profile from app.constants import POST_TYPE_LINK, POST_TYPE_IMAGE, SUBSCRIPTION_MEMBER from app.models import User, Community, CommunityJoinRequest, CommunityMember, CommunityBan, ActivityPubLog, Post, \ - PostReply, Instance, PostVote, PostReplyVote, File, AllowedInstances, BannedInstances, utcnow, Site + PostReply, Instance, PostVote, PostReplyVote, File, AllowedInstances, BannedInstances, utcnow, Site, Notification from app.activitypub.util import public_key, users_total, active_half_year, active_month, local_posts, local_comments, \ post_to_activity, find_actor_or_create, default_context, instance_blocked, find_reply_parent, find_liked_object, \ lemmy_site_data, instance_weight, is_activitypub_request, downvote_post_reply, downvote_post, upvote_post_reply, \ @@ -18,8 +18,6 @@ from app.utils import gibberish, get_setting, is_image_url, allowlist_html, html domain_from_url, markdown_to_html, community_membership, ap_datetime, markdown_to_text import werkzeug.exceptions -INBOX = [] - @bp.route('/.well-known/webfinger') def webfinger(): @@ -402,11 +400,29 @@ def process_inbox_request(request_json, activitypublog_id): else: post.type = POST_TYPE_LINK domain = domain_from_url(post.url) - if not domain.banned: - post.domain_id = domain.id - else: + # notify about links to banned websites. + already_notified = set() # often admins and mods are the same people - avoid notifying them twice + if domain.notify_mods: + for community_member in post.community.moderators(): + notify = Notification(title='Suspicious content', url=post.ap_id, + user_id=community_member.user_id, + author_id=user.id) + db.session.add(notify) + already_notified.add(community_member.user_id) + if domain.notify_admins: + for admin in Site.admins(): + if admin.id not in already_notified: + notify = Notification(title='Suspicious content', + url=post.ap_id, user_id=admin.id, + author_id=user.id) + db.session.add(notify) + if domain.banned: post = None activity_log.exception_message = domain.name + ' is blocked by admin' + if not domain.banned: + domain.post_count += 1 + post.domain = domain + if 'image' in request_json['object']: image = File(source_url=request_json['object']['image']['url']) db.session.add(image) @@ -516,23 +532,41 @@ def process_inbox_request(request_json, activitypublog_id): else: post.type = POST_TYPE_LINK domain = domain_from_url(post.url) - if not domain.banned: - post.domain_id = domain.id - else: + # notify about links to banned websites. + already_notified = set() # often admins and mods are the same people - avoid notifying them twice + if domain.notify_mods: + for community_member in post.community.moderators(): + notify = Notification(title='Suspicious content', url=post.ap_id, + user_id=community_member.user_id, + author_id=user.id) + db.session.add(notify) + already_notified.add(community_member.user_id) + if domain.notify_admins: + for admin in Site.admins(): + if admin.id not in already_notified: + notify = Notification(title='Suspicious content', + url=post.ap_id, user_id=admin.id, + author_id=user.id) + db.session.add(notify) + if domain.banned: post = None activity_log.exception_message = domain.name + ' is blocked by admin' - if 'image' in request_json['object']['object']: + if not domain.banned: + domain.post_count += 1 + post.domain = domain + + if 'image' in request_json['object']['object'] and post: image = File(source_url=request_json['object']['object']['image']['url']) db.session.add(image) post.image = image - if post is not None: - db.session.add(post) - community.post_count += 1 - activity_log.result = 'success' - db.session.commit() - if post.image_id: - make_image_sizes(post.image_id, 266, None, 'posts') + if post is not None: + db.session.add(post) + community.post_count += 1 + activity_log.result = 'success' + db.session.commit() + if post.image_id: + make_image_sizes(post.image_id, 266, None, 'posts') else: post_id, parent_comment_id, root_id = find_reply_parent(in_reply_to) if post_id or parent_comment_id or root_id: @@ -924,6 +958,8 @@ def process_inbox_request(request_json, activitypublog_id): # Flush the caches of any major object that was created. To be sure. if 'user' in vars() and user is not None: user.flush_cache() + if user.instance_id: + user.instance.last_seen = utcnow() # if 'community' in vars() and community is not None: # community.flush_cache() if 'post' in vars() and post is not None: @@ -1095,7 +1131,7 @@ def comment_ap(comment_id): return resp else: reply = PostReply.query.get(comment_id) - continue_discussion(reply.post.id, comment_id) + return continue_discussion(reply.post.id, comment_id) @bp.route('/post/', methods=['GET', 'POST']) diff --git a/app/activitypub/util.py b/app/activitypub/util.py index 378027d2..8eb4d974 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -9,7 +9,7 @@ from flask import current_app, request, g from sqlalchemy import text from app import db, cache, constants, celery from app.models import User, Post, Community, BannedInstances, File, PostReply, AllowedInstances, Instance, utcnow, \ - PostVote, PostReplyVote, ActivityPubLog + PostVote, PostReplyVote, ActivityPubLog, Notification, Site import time import base64 import requests @@ -439,12 +439,26 @@ def post_json_to_model(post_json, user, community) -> Post: post.type = POST_TYPE_IMAGE else: post.type = POST_TYPE_LINK + domain = domain_from_url(post.url) - if not domain.banned: - post.domain_id = domain.id - else: + # notify about links to banned websites. + already_notified = set() # often admins and mods are the same people - avoid notifying them twice + if domain.notify_mods: + for community_member in post.community.moderators(): + notify = Notification(title='Suspicious content', url=post.ap_id, user_id=community_member.user_id, author_id=user.id) + db.session.add(notify) + already_notified.add(community_member.user_id) + if domain.notify_admins: + for admin in Site.admins(): + if admin.id not in already_notified: + notify = Notification(title='Suspicious content', url=post.ap_id, user_id=admin.id, author_id=user.id) + db.session.add(notify) + if domain.banned: post = None - if 'image' in post_json: + if not domain.banned: + domain.post_count += 1 + post.domain = domain + if 'image' in post_json and post: image = File(source_url=post_json['image']['url']) db.session.add(image) post.image = image @@ -620,7 +634,7 @@ def find_instance_id(server): db.session.add(new_instance) db.session.commit() - # Spawn background task + # Spawn background task to fill in more details refresh_instance_profile(new_instance.id) return new_instance.id diff --git a/app/auth/routes.py b/app/auth/routes.py index 684950cd..d240e41f 100644 --- a/app/auth/routes.py +++ b/app/auth/routes.py @@ -76,7 +76,7 @@ def register(): verification_token = random_token(16) form.user_name.data = form.user_name.data.strip() user = User(user_name=form.user_name.data, email=form.real_email.data, - verification_token=verification_token) + verification_token=verification_token, instance=1) user.set_password(form.password.data) db.session.add(user) db.session.commit() diff --git a/app/models.py b/app/models.py index 17b2260c..67c1c340 100644 --- a/app/models.py +++ b/app/models.py @@ -264,6 +264,7 @@ class User(UserMixin, db.Model): avatar = db.relationship('File', lazy='joined', foreign_keys=[avatar_id], single_parent=True, cascade="all, delete-orphan") cover = db.relationship('File', lazy='joined', foreign_keys=[cover_id], single_parent=True, cascade="all, delete-orphan") + instance = db.relationship('Instance', lazy='joined', foreign_keys=[instance_id]) ap_id = db.Column(db.String(255), index=True) # e.g. username@server ap_profile_id = db.Column(db.String(255), index=True) # e.g. https://server/u/username @@ -665,6 +666,8 @@ class Domain(db.Model): name = db.Column(db.String(255), index=True) post_count = db.Column(db.Integer, default=0) banned = db.Column(db.Boolean, default=False, index=True) # Domains can be banned site-wide (by admin) or DomainBlock'ed by users + notify_mods = db.Column(db.Boolean, default=False, index=True) + notify_admins = db.Column(db.Boolean, default=False, index=True) class DomainBlock(db.Model): @@ -738,14 +741,20 @@ class Instance(db.Model): version = db.Column(db.String(50)) created_at = db.Column(db.DateTime, default=utcnow) updated_at = db.Column(db.DateTime, default=utcnow) + last_seen = db.Column(db.DateTime, default=utcnow) # When an Activity was received from them + last_successful_send = db.Column(db.DateTime) # When we successfully sent them an Activity + failures = db.Column(db.Integer, default=0) # How many times we failed to send (reset to 0 after every successful send) + most_recent_attempt = db.Column(db.DateTime) # When the most recent failure was + dormant = db.Column(db.Boolean, default=False) # True once this instance is considered offline and not worth sending to any more + start_trying_again = db.Column(db.DateTime) # When to start trying again. Should grow exponentially with each failure. + gone_forever = db.Column(db.Boolean, default=False) # True once this instance is considered offline forever - never start trying again posts = db.relationship('Post', backref='instance', lazy='dynamic') post_replies = db.relationship('PostReply', backref='instance', lazy='dynamic') communities = db.relationship('Community', backref='instance', lazy='dynamic') - def alive(self): - # todo: determine aliveness based on number of failed connection attempts, etc - return True + def online(self): + return not self.dormant and not self.gone_forever class InstanceBlock(db.Model): @@ -841,8 +850,8 @@ class Notification(db.Model): title = db.Column(db.String(50)) url = db.Column(db.String(512)) read = db.Column(db.Boolean, default=False) - user_id = db.Column(db.Integer, db.ForeignKey('user.id')) - author_id = db.Column(db.Integer, db.ForeignKey('user.id')) + user_id = db.Column(db.Integer, db.ForeignKey('user.id')) # who the notification should go to + author_id = db.Column(db.Integer, db.ForeignKey('user.id')) # the person who caused the notification to happen created_at = db.Column(db.DateTime, default=utcnow) @@ -886,6 +895,10 @@ class Site(db.Model): updated = db.Column(db.DateTime, default=utcnow) last_active = db.Column(db.DateTime, default=utcnow) + @staticmethod + def admins() -> List[User]: + return User.query.filter_by(deleted=False, banned=False).join(user_role).filter(user_role.c.role_id == 4).all() + @login.user_loader def load_user(id): diff --git a/app/user/routes.py b/app/user/routes.py index f3b151b8..87a150a0 100644 --- a/app/user/routes.py +++ b/app/user/routes.py @@ -261,7 +261,7 @@ def send_deletion_requests(user_id): "type": "Delete" } for instance in instances: - if instance.inbox and instance.alive() and instance.id != 1: # instance id 1 is always the current instance + if instance.inbox and instance.online() and instance.id != 1: # instance id 1 is always the current instance post_request(instance.inbox, payload, user.private_key, f"{user.profile_id()}#main-key") sleep(5) diff --git a/migrations/versions/b58c4301d1ad_link_sharing_monitoring.py b/migrations/versions/b58c4301d1ad_link_sharing_monitoring.py new file mode 100644 index 00000000..72fcd52d --- /dev/null +++ b/migrations/versions/b58c4301d1ad_link_sharing_monitoring.py @@ -0,0 +1,38 @@ +"""link sharing monitoring + +Revision ID: b58c4301d1ad +Revises: ea5650ac4628 +Create Date: 2023-12-30 08:08:57.954434 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'b58c4301d1ad' +down_revision = 'ea5650ac4628' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('domain', schema=None) as batch_op: + batch_op.add_column(sa.Column('notify_mods', sa.Boolean(), nullable=True)) + batch_op.add_column(sa.Column('notify_admins', sa.Boolean(), nullable=True)) + batch_op.create_index(batch_op.f('ix_domain_notify_admins'), ['notify_admins'], unique=False) + batch_op.create_index(batch_op.f('ix_domain_notify_mods'), ['notify_mods'], unique=False) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('domain', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_domain_notify_mods')) + batch_op.drop_index(batch_op.f('ix_domain_notify_admins')) + batch_op.drop_column('notify_admins') + batch_op.drop_column('notify_mods') + + # ### end Alembic commands ### diff --git a/migrations/versions/ea5650ac4628_instance_health.py b/migrations/versions/ea5650ac4628_instance_health.py new file mode 100644 index 00000000..011bb26c --- /dev/null +++ b/migrations/versions/ea5650ac4628_instance_health.py @@ -0,0 +1,44 @@ +"""instance health + +Revision ID: ea5650ac4628 +Revises: 88d210da7f2b +Create Date: 2023-12-29 20:26:42.527252 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'ea5650ac4628' +down_revision = '88d210da7f2b' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('instance', schema=None) as batch_op: + batch_op.add_column(sa.Column('last_seen', sa.DateTime(), nullable=True)) + batch_op.add_column(sa.Column('last_successful_send', sa.DateTime(), nullable=True)) + batch_op.add_column(sa.Column('failures', sa.Integer(), nullable=True)) + batch_op.add_column(sa.Column('most_recent_attempt', sa.DateTime(), nullable=True)) + batch_op.add_column(sa.Column('dormant', sa.Boolean(), nullable=True)) + batch_op.add_column(sa.Column('start_trying_again', sa.DateTime(), nullable=True)) + batch_op.add_column(sa.Column('gone_forever', sa.Boolean(), nullable=True)) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('instance', schema=None) as batch_op: + batch_op.drop_column('gone_forever') + batch_op.drop_column('start_trying_again') + batch_op.drop_column('dormant') + batch_op.drop_column('most_recent_attempt') + batch_op.drop_column('failures') + batch_op.drop_column('last_successful_send') + batch_op.drop_column('last_seen') + + # ### end Alembic commands ###