ignore mastodon delete spam and lemmy retry spam

app/activitypub/routes.py

@@ -1,5 +1,3 @@
-from datetime import datetime
-
 from app import db, constants, cache
 from app.activitypub import bp
 from flask import request, Response, current_app, abort, jsonify, json, g
@@ -14,7 +12,7 @@ from app.models import User, Community, CommunityJoinRequest, CommunityMember, C
 from app.activitypub.util import public_key, users_total, active_half_year, active_month, local_posts, local_comments, \
     post_to_activity, find_actor_or_create, default_context, instance_blocked, find_reply_parent, find_liked_object, \
     lemmy_site_data, instance_weight, is_activitypub_request, downvote_post_reply, downvote_post, upvote_post_reply, \
-    upvote_post
+    upvote_post, activity_already_ingested
 from app.utils import gibberish, get_setting, is_image_url, allowlist_html, html_to_markdown, render_template, \
     domain_from_url, markdown_to_html, community_membership, ap_datetime
 import werkzeug.exceptions
@@ -279,11 +277,28 @@ def shared_inbox():
             activity_log.result = 'failure'
             db.session.add(activity_log)
             db.session.commit()
-            return
+            return ''
         else:
             if 'id' in request_json:
                 activity_log.activity_id = request_json['id']
 
+                if activity_already_ingested(request_json['id']):   # Lemmy has an extremely short POST timeout and tends to retry unnecessarily. Ignore their retries.
+                    activity_log.result = 'ignored'
+                    db.session.add(activity_log)
+                    db.session.commit()
+                    return ''
+
+                activity_log.activity_id = request_json['id']
+                activity_log.activity_json = json.dumps(request_json)
+
+                # Mastodon spams the whole fediverse whenever any of their users are deleted. Ignore them, for now. The Activity includes the Actor signature so it should be possible to verify the POST and do the delete if valid, without a call to find_actor_or_create() and all the network activity that involves. One day.
+                if 'type' in request_json and request_json['type'] == 'Delete' and request_json['id'].endswith('#delete'):
+                    activity_log.result = 'ignored'
+                    activity_log.activity_type = 'Delete'
+                    db.session.add(activity_log)
+                    db.session.commit()
+                    return ''
+
         actor = find_actor_or_create(request_json['actor']) if 'actor' in request_json else None
         if actor is not None:
             if HttpSignature.verify_request(request, actor.public_key, skip_date=True):
@@ -304,7 +319,7 @@ def shared_inbox():
                                     community_ap_id = request_json['object']['cc'][0]
                             community = find_actor_or_create(community_ap_id)
                             user = find_actor_or_create(user_ap_id)
-                            if user and community:
+                            if (user and not user.is_local()) and community:
                                 user.last_seen = community.last_active = g.site.last_active = utcnow()
 
                                 object_type = request_json['object']['type']
@@ -401,7 +416,8 @@ def shared_inbox():
                                                 community.last_active = post.last_active = utcnow()
                                                 activity_log.result = 'success'
                                                 db.session.commit()
-                                                vote = PostReplyVote(user_id=user.id, author_id=post_reply.user_id, post_reply_id=post_reply.id,
+                                                vote = PostReplyVote(user_id=user.id, author_id=post_reply.user_id,
+                                                                     post_reply_id=post_reply.id,
                                                                      effect=instance_weight(user.ap_domain))
                                                 db.session.add(vote)
                                             else:
@@ -597,7 +613,7 @@ def shared_inbox():
                                                                     exception_message='could not send Accept' + str(e))
                                         db.session.add(accept_log)
                                         db.session.commit()
-                                        return
+                                        return ''
                                     activity_log.result = 'success'
                                 else:
                                     activity_log.exception_message = 'user is banned from this community'
@@ -804,8 +820,6 @@ def shared_inbox():
     return ''
 
 
-
-
 @bp.route('/c/<actor>/outbox', methods=['GET'])
 def community_outbox(actor):
     actor = actor.strip()
@@ -927,6 +941,7 @@ def post_ap(post_id):
 
 
 @bp.route('/activities/<type>/<id>')
+@cache.cached(timeout=600)
 def activities_json(type, id):
     activity = ActivityPubLog.query.filter_by(activity_id=f"https://{current_app.config['SERVER_NAME']}/activities/{type}/{id}").first()
     if activity:

app/activitypub/util.py

@@ -3,11 +3,11 @@ from __future__ import annotations
 import json
 import os
 from typing import Union, Tuple
-from flask import current_app, request
+from flask import current_app, request, g
 from sqlalchemy import text
 from app import db, cache, constants
 from app.models import User, Post, Community, BannedInstances, File, PostReply, AllowedInstances, Instance, utcnow, \
-    Site, PostVote, PostReplyVote
+    Site, PostVote, PostReplyVote, ActivityPubLog
 import time
 import base64
 import requests
@@ -504,6 +504,11 @@ def is_activitypub_request():
     return 'application/ld+json' in request.headers.get('Accept', '') or 'application/activity+json' in request.headers.get('Accept', '')
 
 
+def activity_already_ingested(ap_id):
+    return db.session.execute(text('SELECT id FROM "activity_pub_log" WHERE activity_id = :activity_id'),
+                              {'activity_id': ap_id}).scalar()
+
+
 def downvote_post(post, user):
     user.last_seen = utcnow()
     existing_vote = PostVote.query.filter_by(user_id=user.id, post_id=post.id).first()
@@ -627,7 +632,7 @@ def upvote_post(post, user):
 
 
 def lemmy_site_data():
-    site = Site.query.get(1)
+    site = g.site
     data = {
       "site_view": {
         "site": {

app/admin/routes.py

@@ -1,6 +1,6 @@
 from datetime import datetime, timedelta
 
-from flask import request, flash
+from flask import request, flash, json, url_for
 from flask_login import login_required, current_user
 from flask_babel import _
 from sqlalchemy import text, desc
@@ -126,5 +126,23 @@ def admin_activities():
         ActivityPubLog.created_at < utcnow() - timedelta(days=3)).delete()
     db.session.commit()
 
-    return render_template('admin/activities.html', title=_('ActivityPub Log'),
-                           activities=ActivityPubLog.query.order_by(desc(ActivityPubLog.created_at)).all())
+    page = request.args.get('page', 1, type=int)
+
+    activities = ActivityPubLog.query.order_by(desc(ActivityPubLog.created_at)).paginate(page=page, per_page=1000, error_out=False)
+
+    next_url = url_for('admin.admin_activities',
+                       page=activities.next_num) if activities.has_next else None
+    prev_url = url_for('admin.admin_activities',
+                       page=activities.prev_num) if activities.has_prev and page != 1 else None
+
+    return render_template('admin/activities.html', title=_('ActivityPub Log'), next_url=next_url, prev_url=prev_url,
+                           activities=activities)
+
+
+@bp.route('/activity_json/<int:activity_id>')
+@login_required
+@permission_required('change instance settings')
+def activity_json(activity_id):
+    activity = ActivityPubLog.query.get_or_404(activity_id)
+    return render_template('admin/activity_json.html', title=_('Activity JSON'),
+                           activity_json_data=json.loads(activity.activity_json))