From fe7791bf4df7b77656386fcc50b7a2308e80dd37 Mon Sep 17 00:00:00 2001 From: freamon Date: Mon, 18 Nov 2024 15:56:47 +0000 Subject: [PATCH 01/36] apf part 01: add log_incoming_ap function --- app/activitypub/routes.py | 2 +- app/activitypub/util.py | 13 +++++++++++++ app/constants.py | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 1 deletion(-) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index b82cdf27..a5f91568 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -25,7 +25,7 @@ from app.activitypub.util import public_key, users_total, active_half_year, acti update_post_from_activity, undo_vote, undo_downvote, post_to_page, get_redis_connection, find_reported_object, \ process_report, ensure_domains_match, can_edit, can_delete, remove_data_from_banned_user, resolve_remote_post, \ inform_followers_of_post_update, comment_model_to_json, restore_post_or_comment, ban_local_user, unban_local_user, \ - lock_post + lock_post, log_incoming_ap from app.utils import gibberish, get_setting, render_template, \ community_membership, ap_datetime, ip_address, can_downvote, \ can_upvote, can_create_post, awaken_dormant_instance, shorten_string, can_create_post_reply, sha256_digest, \ diff --git a/app/activitypub/util.py b/app/activitypub/util.py index f0811f7a..e4e92f1f 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -2565,3 +2565,16 @@ def inform_followers_of_post_update_task(post_id: int, sending_instance_id: int) post_request(i.inbox, update_json, post.author.private_key, post.author.public_url() + '#main-key') except Exception: pass + + +def log_incoming_ap(id, aplog_type, aplog_result, request_json, message=None): + aplog_in = APLOG_IN + + if aplog_in and aplog_type[0] and aplog_result[0]: + activity_log = ActivityPubLog(direction='in', activity_id=id, activity_type=aplog_type[1], result=aplog_result[1]) + if message: + activity_log.exception_message = message + if request_json: + activity_log.activity_json = json.dumps(request_json) + db.session.add(activity_log) + db.session.commit() diff --git a/app/constants.py b/app/constants.py index e6897458..b53aa12a 100644 --- a/app/constants.py +++ b/app/constants.py @@ -36,3 +36,37 @@ ROLE_STAFF = 3 ROLE_ADMIN = 4 MICROBLOG_APPS = ["mastodon", "misskey", "akkoma", "iceshrimp", "pleroma"] + +APLOG_IN = True + +APLOG_MONITOR = (True, 'Debug this') + +APLOG_SUCCESS = (True, 'success') +APLOG_FAILURE = (True, 'failure') +APLOG_IGNORED = (True, 'ignored') +APLOG_PROCESSING = (True, 'processing') + +APLOG_NOTYPE = (True, 'Unknown') +APLOG_DUPLICATE = (True, 'Duplicate') +APLOG_FOLLOW = (True, 'Follow') +APLOG_ACCEPT = (True, 'Accept') +APLOG_DELETE = (True, 'Delete') +APLOG_CHATMESSAGE = (True, 'Create ChatMessage') +APLOG_CREATE = (True, 'Create') +APLOG_UPDATE = (True, 'Update') +APLOG_LIKE = (True, 'Like') +APLOG_DISLIKE = (True, 'Dislike') +APLOG_REPORT = (True, 'Report') +APLOG_USERBAN = (True, 'User Ban') +APLOG_LOCK = (True, 'Post Lock') + +APLOG_UNDO_FOLLOW = (True, 'Undo Follow') +APLOG_UNDO_DELETE = (True, 'Undo Delete') +APLOG_UNDO_VOTE = (True, 'Undo Vote') +APLOG_UNDO_USERBAN = (True, 'Undo User Ban') + +APLOG_ADD = (True, 'Add Mod/Sticky') +APLOG_REMOVE = (True, 'Remove Mod/Sticky') + +APLOG_ANNOUNCE = (True, 'Announce') +APLOG_PT_VIEW = (True, 'PeerTube View') From cddd102a3df3cd3ab61034ecda8e387d9e43da50 Mon Sep 17 00:00:00 2001 From: freamon Date: Mon, 18 Nov 2024 16:53:32 +0000 Subject: [PATCH 02/36] apf part 02: reject bad JSONs --- app/activitypub/routes.py | 48 +++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index a5f91568..8d582b1c 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -14,7 +14,7 @@ from app.community.routes import show_community from app.community.util import send_to_remote_instance from app.post.routes import continue_discussion, show_post from app.user.routes import show_profile -from app.constants import POST_TYPE_LINK, POST_TYPE_IMAGE, SUBSCRIPTION_MEMBER +from app.constants import * from app.models import User, Community, CommunityJoinRequest, CommunityMember, CommunityBan, ActivityPubLog, Post, \ PostReply, Instance, PostVote, PostReplyVote, File, AllowedInstances, BannedInstances, utcnow, Site, Notification, \ ChatMessage, Conversation, UserFollower, UserBlock, Poll, PollChoice @@ -390,21 +390,39 @@ def community_profile(actor): abort(404) -@bp.route('/inbox', methods=['GET', 'POST']) +@bp.route('/inbox', methods=['POST']) def shared_inbox(): + try: + request_json = request.get_json(force=True) + except werkzeug.exceptions.BadRequest as e: + log_incoming_ap('', APLOG_NOTYPE, APLOG_FAILURE, None, 'Unable to parse json body: ' + e.description) + return '', 400 + + g.site = Site.query.get(1) # g.site is not initialized by @app.before_request when request.path == '/inbox' + store_ap_json = g.site.log_activitypub_json + + if not 'id' in request_json or not 'type' in request_json or not 'actor' in request_json or not 'object' in request_json: + log_incoming_ap('', APLOG_NOTYPE, APLOG_FAILURE, request_json if store_ap_json else None, 'Missing minimum expected fields in JSON') + return '', 400 + + id = request_json['id'] + if request_json['type'] == 'Announce' and isinstance(request_json['object'], dict): + object = request_json['object'] + if not 'id' in object or not 'type' in object or not 'actor' in object or not 'object' in object: + if 'type' in object and (object['type'] == 'Page' or object['type'] == 'Note'): + log_incoming_ap(request_json['id'], APLOG_ANNOUNCE, APLOG_IGNORED, request_json if store_ap_json else None, 'Intended for Mastodon') + else: + log_incoming_ap(request_json['id'], APLOG_ANNOUNCE, APLOG_FAILURE, request_json if store_ap_json else None, 'Missing minimum expected fields in JSON Announce object') + return '', 400 + + if object['actor'].startswith('https://' + current_app.config['SERVER_NAME']): + log_incoming_ap(object['id'], APLOG_DUPLICATE, APLOG_IGNORED, request_json if store_ap_json else None, 'Activity about local content which is already present') + return '', 400 + if request.method == 'POST': # save all incoming data to aid in debugging and development. Set result to 'success' if things go well activity_log = ActivityPubLog(direction='in', result='failure') - try: - request_json = request.get_json(force=True) - except werkzeug.exceptions.BadRequest as e: - activity_log.exception_message = 'Unable to parse json body: ' + e.description - activity_log.result = 'failure' - db.session.add(activity_log) - db.session.commit() - return '' - if 'id' in request_json: redis_client = get_redis_connection() if redis_client.get(request_json['id']) is not None: # Lemmy has an extremely short POST timeout and tends to retry unnecessarily. Ignore their retries. @@ -416,7 +434,6 @@ def shared_inbox(): redis_client.set(request_json['id'], 1, ex=90) # Save the activity ID into redis, to avoid duplicate activities that Lemmy sometimes sends activity_log.activity_id = request_json['id'] - g.site = Site.query.get(1) # g.site is not initialized by @app.before_request when request.path == '/inbox' if g.site.log_activitypub_json: activity_log.activity_json = json.dumps(request_json) activity_log.result = 'processing' @@ -438,13 +455,6 @@ def shared_inbox(): db.session.commit() return '' - else: - activity_log.activity_id = '' - if g.site.log_activitypub_json: - activity_log.activity_json = json.dumps(request_json) - db.session.add(activity_log) - db.session.commit() - actor = find_actor_or_create(request_json['actor']) if 'actor' in request_json else None if actor is not None: try: From 97af03d3cfec94343795ca023195048cc6d4ee4a Mon Sep 17 00:00:00 2001 From: freamon Date: Mon, 18 Nov 2024 17:16:46 +0000 Subject: [PATCH 03/36] apf part 03: reject unneccessary retries --- app/activitypub/routes.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index 8d582b1c..34cde840 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -419,20 +419,17 @@ def shared_inbox(): log_incoming_ap(object['id'], APLOG_DUPLICATE, APLOG_IGNORED, request_json if store_ap_json else None, 'Activity about local content which is already present') return '', 400 + redis_client = get_redis_connection() + if redis_client.exists(id): # Something is sending same activity multiple times, or Announcing as well as sending the same content + log_incoming_ap(id, APLOG_DUPLICATE, APLOG_IGNORED, request_json if store_ap_json else None, 'Unnecessary retry attempt') + return '', 400 + redis_client.set(id, 1, ex=90) # Save the activity ID into redis, to avoid duplicate activities + if request.method == 'POST': # save all incoming data to aid in debugging and development. Set result to 'success' if things go well activity_log = ActivityPubLog(direction='in', result='failure') if 'id' in request_json: - redis_client = get_redis_connection() - if redis_client.get(request_json['id']) is not None: # Lemmy has an extremely short POST timeout and tends to retry unnecessarily. Ignore their retries. - activity_log.result = 'ignored' - activity_log.exception_message = 'Unnecessary retry attempt' - db.session.add(activity_log) - db.session.commit() - return '' - - redis_client.set(request_json['id'], 1, ex=90) # Save the activity ID into redis, to avoid duplicate activities that Lemmy sometimes sends activity_log.activity_id = request_json['id'] if g.site.log_activitypub_json: activity_log.activity_json = json.dumps(request_json) From e57ca1428ed2136c3bc4eff23fc3e8e550731f73 Mon Sep 17 00:00:00 2001 From: freamon Date: Mon, 18 Nov 2024 17:18:09 +0000 Subject: [PATCH 04/36] apf part 04: ignore unutilised PeerTube activity --- app/activitypub/routes.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index 34cde840..53e34a92 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -425,6 +425,11 @@ def shared_inbox(): return '', 400 redis_client.set(id, 1, ex=90) # Save the activity ID into redis, to avoid duplicate activities + # Ignore unutilised PeerTube activity + if request_json['actor'].endswith('accounts/peertube'): + log_incoming_ap(request_json['id'], APLOG_PT_VIEW, APLOG_IGNORED, request_json if store_ap_json else None, 'PeerTube View or CacheFile activity') + return '' + if request.method == 'POST': # save all incoming data to aid in debugging and development. Set result to 'success' if things go well activity_log = ActivityPubLog(direction='in', result='failure') @@ -444,13 +449,6 @@ def shared_inbox(): else: process_delete_request.delay(request_json, activity_log.id, ip_address()) return '' - # Ignore unutilised PeerTube activity - if 'actor' in request_json and request_json['actor'].endswith('accounts/peertube'): - activity_log.result = 'ignored' - activity_log.exception_message = 'PeerTube View or CacheFile activity' - db.session.add(activity_log) - db.session.commit() - return '' actor = find_actor_or_create(request_json['actor']) if 'actor' in request_json else None if actor is not None: From 20f17000f5faadd16e4fb7ee09123b89afb62201 Mon Sep 17 00:00:00 2001 From: freamon Date: Mon, 18 Nov 2024 17:28:41 +0000 Subject: [PATCH 05/36] apf part 05: ignore self-deletes from users that do not exist here --- app/activitypub/routes.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index 53e34a92..52191a35 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -430,6 +430,18 @@ def shared_inbox(): log_incoming_ap(request_json['id'], APLOG_PT_VIEW, APLOG_IGNORED, request_json if store_ap_json else None, 'PeerTube View or CacheFile activity') return '' + # Ignore delete requests from uses that do not already exist here + if request_json['type'] == 'Delete': + if (request_json['id'].endswith('#delete') or # Mastodon / PieFed + ('object' in request_json and isinstance(request_json['object'], str) and request_json['actor'] == request_json['object'])): # Lemmy + actor = User.query.filter_by(ap_profile_id=request_json['actor'].lower()).first() + if not actor: + log_incoming_ap(request_json['id'], APLOG_DELETE, APLOG_IGNORED, request_json if store_ap_json else None, 'Does not exist here') + return '', 400 + else: + actor.ap_fetched_at = utcnow() # use stored pubkey, don't try to re-fetch for next step (signature verification) + db.session.commit() + if request.method == 'POST': # save all incoming data to aid in debugging and development. Set result to 'success' if things go well activity_log = ActivityPubLog(direction='in', result='failure') @@ -1308,9 +1320,6 @@ def process_delete_request(request_json, activitypublog_id, ip_address): else: activity_log.result = 'ignored' activity_log.exception_message = 'Only remote users can be deleted remotely' - else: - activity_log.result = 'ignored' - activity_log.exception_message = 'Does not exist here' db.session.commit() From 479bbe4dcc5b5de3676c7a0b17c9d26f49e9bd96 Mon Sep 17 00:00:00 2001 From: freamon Date: Mon, 18 Nov 2024 17:47:26 +0000 Subject: [PATCH 06/36] apf part 06: reject any ActivityPub activity from a local actor --- app/activitypub/routes.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index 52191a35..6bcd1e21 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -442,6 +442,23 @@ def shared_inbox(): actor.ap_fetched_at = utcnow() # use stored pubkey, don't try to re-fetch for next step (signature verification) db.session.commit() + actor = find_actor_or_create(request_json['actor']) + if not actor: + actor_name = request_json['actor'] + log_incoming_ap(request_json['id'], APLOG_NOTYPE, APLOG_FAILURE, request_json if store_ap_json else None, f'Actor could not be found 1: {actor_name}') + return '', 400 + + if actor.is_local(): # should be impossible (can be Announced back, but not sent without access to privkey) + log_incoming_ap(request_json['id'], APLOG_NOTYPE, APLOG_FAILURE, request_json if store_ap_json else None, 'ActivityPub activity from a local actor') + return '', 400 + else: + actor.instance.last_seen = utcnow() + actor.instance.dormant = False + actor.instance.gone_forever = False + actor.instance.failures = 0 + actor.instance.ip_address = ip_address() + db.session.commit() + if request.method == 'POST': # save all incoming data to aid in debugging and development. Set result to 'success' if things go well activity_log = ActivityPubLog(direction='in', result='failure') @@ -462,7 +479,6 @@ def shared_inbox(): process_delete_request.delay(request_json, activity_log.id, ip_address()) return '' - actor = find_actor_or_create(request_json['actor']) if 'actor' in request_json else None if actor is not None: try: HttpSignature.verify_request(request, actor.public_key, skip_date=True) @@ -476,9 +492,6 @@ def shared_inbox(): activity_log.result = 'failure' db.session.commit() return '', 400 - else: - actor_name = request_json['actor'] if 'actor' in request_json else '' - activity_log.exception_message = f'Actor could not be found 1: {actor_name}' if activity_log.exception_message is not None: activity_log.result = 'failure' From 7e873086635e32a4a857fa80dbe3109614a3fe91 Mon Sep 17 00:00:00 2001 From: freamon Date: Mon, 18 Nov 2024 18:44:28 +0000 Subject: [PATCH 07/36] apf part 07: verify LD signature if HTTP signature fails --- app/activitypub/routes.py | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index 6bcd1e21..726d4eab 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -9,7 +9,7 @@ import werkzeug.exceptions from app import db, constants, cache, celery from app.activitypub import bp -from app.activitypub.signature import HttpSignature, post_request, VerificationError, default_context +from app.activitypub.signature import HttpSignature, post_request, VerificationError, default_context, LDSignature from app.community.routes import show_community from app.community.util import send_to_remote_instance from app.post.routes import continue_discussion, show_post @@ -459,6 +459,19 @@ def shared_inbox(): actor.instance.ip_address = ip_address() db.session.commit() + try: + HttpSignature.verify_request(request, actor.public_key, skip_date=True) + except VerificationError as e: + if not 'signature' in request_json: + log_incoming_ap(request_json['id'], APLOG_NOTYPE, APLOG_FAILURE, request_json if store_ap_json else None, 'Could not verify HTTP signature: ' + str(e)) + return '', 400 + # HTTP sig will fail if a.gup.pe or PeerTube have bounced a request, so check LD sig instead + try: + LDSignature.verify_signature(request_json, actor.public_key) + except VerificationError as e: + log_incoming_ap(request_json['id'], APLOG_NOTYPE, APLOG_FAILURE, request_json if store_ap_json else None, 'Could not verify LD signature: ' + str(e)) + return '', 400 + if request.method == 'POST': # save all incoming data to aid in debugging and development. Set result to 'success' if things go well activity_log = ActivityPubLog(direction='in', result='failure') @@ -480,18 +493,11 @@ def shared_inbox(): return '' if actor is not None: - try: - HttpSignature.verify_request(request, actor.public_key, skip_date=True) - if current_app.debug: - process_inbox_request(request_json, activity_log.id, ip_address()) - else: - process_inbox_request.delay(request_json, activity_log.id, ip_address()) - return '' - except VerificationError as e: - activity_log.exception_message = 'Could not verify signature: ' + str(e) - activity_log.result = 'failure' - db.session.commit() - return '', 400 + if current_app.debug: + process_inbox_request(request_json, activity_log.id, ip_address()) + else: + process_inbox_request.delay(request_json, activity_log.id, ip_address()) + return '' if activity_log.exception_message is not None: activity_log.result = 'failure' From eb7095af5624507bea442a57d6447fec78218cd1 Mon Sep 17 00:00:00 2001 From: freamon Date: Mon, 18 Nov 2024 19:28:37 +0000 Subject: [PATCH 08/36] apf part 08: process requests from users who want to delete their own account --- app/activitypub/routes.py | 68 ++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 41 deletions(-) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index 726d4eab..4e430aa2 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -472,6 +472,17 @@ def shared_inbox(): log_incoming_ap(request_json['id'], APLOG_NOTYPE, APLOG_FAILURE, request_json if store_ap_json else None, 'Could not verify LD signature: ' + str(e)) return '', 400 + # When a user is deleted, the only way to be fairly sure they get deleted everywhere is to tell the whole fediverse. + # Earlier check means this is only for users that already exist, repeating it here means that http signature will have been verified + if request_json['type'] == 'Delete': + if (request_json['id'].endswith('#delete') or # Mastodon / PieFed + ('object' in request_json and isinstance(request_json['object'], str) and request_json['actor'] == request_json['object'])): # Lemmy + if current_app.debug: + process_delete_request(request_json, store_ap_json) + else: + process_delete_request.delay(request_json, store_ap_json) + return '' + if request.method == 'POST': # save all incoming data to aid in debugging and development. Set result to 'success' if things go well activity_log = ActivityPubLog(direction='in', result='failure') @@ -484,14 +495,6 @@ def shared_inbox(): db.session.add(activity_log) db.session.commit() - # When a user is deleted, the only way to be fairly sure they get deleted everywhere is to tell the whole fediverse. - if 'type' in request_json and request_json['type'] == 'Delete' and request_json['id'].endswith('#delete'): - if current_app.debug: - process_delete_request(request_json, activity_log.id, ip_address()) - else: - process_delete_request.delay(request_json, activity_log.id, ip_address()) - return '' - if actor is not None: if current_app.debug: process_inbox_request(request_json, activity_log.id, ip_address()) @@ -1305,41 +1308,24 @@ def process_inbox_request(request_json, activitypublog_id, ip_address): @celery.task -def process_delete_request(request_json, activitypublog_id, ip_address): +def process_delete_request(request_json, store_ap_json): with current_app.app_context(): - activity_log = ActivityPubLog.query.get(activitypublog_id) - if 'type' in request_json and request_json['type'] == 'Delete': - if isinstance(request_json['object'], dict): - # wafrn sends invalid delete requests - return - else: - actor_to_delete = request_json['object'].lower() - user = User.query.filter_by(ap_profile_id=actor_to_delete).first() - if user: - # check that the user really has been deleted, to avoid spoofing attacks - if not user.is_local(): - if user_removed_from_remote_server(actor_to_delete, is_piefed=user.instance.software == 'PieFed'): - # Delete all their images to save moderators from having to see disgusting stuff. - files = File.query.join(Post).filter(Post.user_id == user.id).all() - for file in files: - file.delete_from_disk() - file.source_url = '' - if user.avatar_id: - user.avatar.delete_from_disk() - user.avatar.source_url = '' - if user.cover_id: - user.cover.delete_from_disk() - user.cover.source_url = '' - user.banned = True - user.deleted = True - activity_log.result = 'success' - else: - activity_log.result = 'ignored' - activity_log.exception_message = 'User not actually deleted.' - else: - activity_log.result = 'ignored' - activity_log.exception_message = 'Only remote users can be deleted remotely' + # this function processes self-deletes (retain case here, as user_removed_from_remote_server() uses a JSON request) + user_ap_id = request_json['actor'] + user = User.query.filter_by(ap_profile_id=user_ap_id.lower()).first() + if user: + # check that the user really has been deleted, to avoid spoofing attacks + if user_removed_from_remote_server(user_ap_id, is_piefed=user.instance.software == 'PieFed'): + # soft self-delete + user.deleted = True + user.deleted_by = user.id db.session.commit() + log_incoming_ap(request_json['id'], APLOG_DELETE, APLOG_SUCCESS, request_json if store_ap_json else None) + else: + log_incoming_ap(request_json['id'], APLOG_DELETE, APLOG_FAILURE, request_json if store_ap_json else None, 'User not actually deleted.') + # TODO: process self-undeletes from Lemmy + # TODO: acknowledge 'removeData' field from Lemmy + # TODO: hard-delete in 7 days (should purge avatar and cover images, but keep posts and replies unless already soft-deleted by removeData = True) def announce_activity_to_followers(community, creator, activity): From 3d053ae70aee975f755b67f572efdb3ff45d1d6b Mon Sep 17 00:00:00 2001 From: freamon Date: Mon, 18 Nov 2024 21:10:12 +0000 Subject: [PATCH 09/36] apf part 09: Follow requests --- app/activitypub/routes.py | 129 +++++++++++++++++++++++++++++++------- 1 file changed, 106 insertions(+), 23 deletions(-) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index 4e430aa2..9e731269 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -483,28 +483,11 @@ def shared_inbox(): process_delete_request.delay(request_json, store_ap_json) return '' - if request.method == 'POST': - # save all incoming data to aid in debugging and development. Set result to 'success' if things go well - activity_log = ActivityPubLog(direction='in', result='failure') + if current_app.debug: + process_inbox_request(request_json, store_ap_json) + else: + process_inbox_request.delay(request_json, store_ap_json) - if 'id' in request_json: - activity_log.activity_id = request_json['id'] - if g.site.log_activitypub_json: - activity_log.activity_json = json.dumps(request_json) - activity_log.result = 'processing' - db.session.add(activity_log) - db.session.commit() - - if actor is not None: - if current_app.debug: - process_inbox_request(request_json, activity_log.id, ip_address()) - else: - process_inbox_request.delay(request_json, activity_log.id, ip_address()) - return '' - - if activity_log.exception_message is not None: - activity_log.result = 'failure' - db.session.commit() return '' @@ -514,10 +497,110 @@ def site_inbox(): @celery.task -def process_inbox_request(request_json, activitypublog_id, ip_address): +def process_inbox_request(request_json, store_ap_json): with current_app.app_context(): - activity_log = ActivityPubLog.query.get(activitypublog_id) site = Site.query.get(1) # can't use g.site because celery doesn't use Flask's g variable + + # For an Announce, Accept, or Reject, we have the community, and need to find the user + # For everything else, we have the user, and need to find the community + # Benefits of always using request_json['actor']: + # It's the actor who signed the request, and whose signature has been verified + # Because of the earlier check, we know that they already exist, and so don't need to check again + # Using actors from inner objects has a vulnerability to spoofing attacks (e.g. if 'attributedTo' doesn't match the 'Create' actor) + + if request_json['type'] == 'Announce' or request_json['type'] == 'Accept' or request_json['type'] == 'Reject': + community_ap_id = request_json['actor'] + community = find_actor_or_create(community_ap_id, community_only=True, create_if_not_found=False) + if not community or not isinstance(community, Community): + log_incoming_ap(announce_id, APLOG_ANNOUNCE, APLOG_FAILURE, request_json, 'Actor was not a community') + return + user_ap_id = None # found in 'if request_json['type'] == 'Announce', or it's a local user (for 'Accept'/'Reject') + else: + user_ap_id = request_json['actor'] + user = find_actor_or_create(user_ap_id, create_if_not_found=False) + if not user or not isinstance(user, User): + log_incoming_ap(announce_id, APLOG_NOTYPE, APLOG_FAILURE, request_json, 'Actor was not a user') + return + user.last_seen = site.last_active = utcnow() + db.session.commit() + community = None # found as needed + + # Follow: remote user wants to join/follow one of our users or communities + if request_json['type'] == 'Follow': + target_ap_id = request_json['object'] + follow_id = request_json['id'] + target = find_actor_or_create(target_ap_id, create_if_not_found=False) + if not target: + log_incoming_ap(request_json['id'], APLOG_FOLLOW, APLOG_FAILURE, request_json if store_ap_json else None, 'Could not find target of Follow') + return + if isinstance(target, Community): + community = target + reject_follow = False + if community.local_only: + log_incoming_ap(request_json['id'], APLOG_FOLLOW, APLOG_FAILURE, request_json if store_ap_json else None, 'Local only cannot be followed by remote users') + reject_follow = True + else: + # check if user is banned from this community + user_banned = CommunityBan.query.filter_by(user_id=user.id, community_id=community.id).first() + if user_banned: + log_incoming_ap(request_json['id'], APLOG_FOLLOW, APLOG_FAILURE, request_json if store_ap_json else None, 'Remote user has been banned') + reject_follow = True + if reject_follow: + # send reject message to deny the follow + reject = {"@context": default_context(), "actor": community.public_url(), "to": [user.public_url()], + "object": {"actor": user.public_url(), "to": None, "object": community.public_url(), "type": "Follow", "id": follow_id}, + "type": "Reject", "id": f"https://{current_app.config['SERVER_NAME']}/activities/reject/" + gibberish(32)} + post_request(user.ap_inbox_url, reject, community.private_key, f"{community.public_url()}#main-key") + else: + if community_membership(user, community) != SUBSCRIPTION_MEMBER: + member = CommunityMember(user_id=user.id, community_id=community.id) + db.session.add(member) + db.session.commit() + cache.delete_memoized(community_membership, user, community) + # send accept message to acknowledge the follow + accept = {"@context": default_context(), "actor": community.public_url(), "to": [user.public_url()], + "object": {"actor": user.public_url(), "to": None, "object": community.public_url(), "type": "Follow", "id": follow_id}, + "type": "Accept", "id": f"https://{current_app.config['SERVER_NAME']}/activities/accept/" + gibberish(32)} + post_request(user.ap_inbox_url, accept, community.private_key, f"{community.public_url()}#main-key") + log_incoming_ap(request_json['id'], APLOG_FOLLOW, APLOG_SUCCESS, request_json if store_ap_json else None) + return + elif isinstance(target, User): + local_user = target + remote_user = user + if not local_user.is_local(): + log_incoming_ap(request_json['id'], APLOG_FOLLOW, APLOG_FAILURE, request_json if store_ap_json else None, 'Follow request for remote user received') + return + existing_follower = UserFollower.query.filter_by(local_user_id=local_user.id, remote_user_id=remote_user.id).first() + if not existing_follower: + auto_accept = not local_user.ap_manually_approves_followers + new_follower = UserFollower(local_user_id=local_user.id, remote_user_id=remote_user.id, is_accepted=auto_accept) + if not local_user.ap_followers_url: + local_user.ap_followers_url = local_user.public_url() + '/followers' + db.session.add(new_follower) + db.session.commit() + accept = {"@context": default_context(), "actor": local_user.public_url(), "to": [remote_user.public_url()], + "object": {"actor": remote_user.public_url(), "to": None, "object": local_user.public_url(), "type": "Follow", "id": follow_id}, + "type": "Accept", "id": f"https://{current_app.config['SERVER_NAME']}/activities/accept/" + gibberish(32)} + post_request(remote_user.ap_inbox_url, accept, local_user.private_key, f"{local_user.public_url()}#main-key") + log_incoming_ap(request_json['id'], APLOG_FOLLOW, APLOG_SUCCESS, request_json if store_ap_json else None) + return + + + + + # -- below this point is code that will be incrementally replaced to use log_incoming_ap() instead -- + + # save all incoming data to aid in debugging and development. Set result to 'success' if things go well + activity_log = ActivityPubLog(direction='in', result='failure') + + if 'id' in request_json: + activity_log.activity_id = request_json['id'] + if site.log_activitypub_json: + activity_log.activity_json = json.dumps(request_json) + activity_log.result = 'processing' + db.session.add(activity_log) + db.session.commit() + if 'type' in request_json: activity_log.activity_type = request_json['type'] if not instance_blocked(request_json['id']): From 3cc411f51203b347be6aacd54598d8f0d7d46577 Mon Sep 17 00:00:00 2001 From: freamon Date: Mon, 18 Nov 2024 21:19:42 +0000 Subject: [PATCH 10/36] apf part 10: Accepts for follows --- app/activitypub/routes.py | 26 ++++++++++++++++++++++++++ app/community/routes.py | 8 ++++---- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index 9e731269..a7365245 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -585,6 +585,32 @@ def process_inbox_request(request_json, store_ap_json): log_incoming_ap(request_json['id'], APLOG_FOLLOW, APLOG_SUCCESS, request_json if store_ap_json else None) return + # Accept: remote server is accepting our previous follow request + if request_json['type'] == 'Accept': + user = None + if isinstance(request_json['object'], str): # a.gup.pe accepts using a string with the ID of the follow request + join_request_parts = request_json['object'].split('/') + join_request = CommunityJoinRequest.query.get(join_request_parts[-1]) + if join_request: + user = User.query.get(join_request.user_id) + elif request_json['object']['type'] == 'Follow': + user_ap_id = request_json['object']['actor'] + user = find_actor_or_create(user_ap_id, create_if_not_found=False) + if not user: + log_incoming_ap(request_json['id'], APLOG_ACCEPT, APLOG_FAILURE, request_json if store_ap_json else None, 'Could not find recipient of Accept') + return + join_request = CommunityJoinRequest.query.filter_by(user_id=user.id, community_id=community.id).first() + if join_request: + existing_membership = CommunityMember.query.filter_by(user_id=join_request.user_id, community_id=join_request.community_id).first() + if not existing_membership: + member = CommunityMember(user_id=join_request.user_id, community_id=join_request.community_id) + db.session.add(member) + community.subscriptions_count += 1 + db.session.commit() + cache.delete_memoized(community_membership, user, community) + log_incoming_ap(request_json['id'], APLOG_ACCEPT, APLOG_SUCCESS, request_json if store_ap_json else None) + return + diff --git a/app/community/routes.py b/app/community/routes.py index 8d202179..3de5b0d0 100644 --- a/app/community/routes.py +++ b/app/community/routes.py @@ -435,6 +435,10 @@ def do_subscribe(actor, user_id, admin_preload=False): else: pre_load_message['community_banned_by_local_instance'] = True success = True + # for local communities, joining is instant + member = CommunityMember(user_id=user.id, community_id=community.id) + db.session.add(member) + db.session.commit() if remote: # send ActivityPub message to remote community, asking to follow. Accept message will be sent to our shared inbox join_request = CommunityJoinRequest(user_id=user.id, community_id=community.id) @@ -464,10 +468,6 @@ def do_subscribe(actor, user_id, admin_preload=False): else: pre_load_message['status'] = msg_to_user - # for local communities, joining is instant - member = CommunityMember(user_id=user.id, community_id=community.id) - db.session.add(member) - db.session.commit() if success is True: if not admin_preload: flash('You joined ' + community.title) From 37462f57de30d23bc2f5506094a8070b831dd152 Mon Sep 17 00:00:00 2001 From: freamon Date: Mon, 18 Nov 2024 21:25:19 +0000 Subject: [PATCH 11/36] apf part 11: Rejects for follows --- app/activitypub/routes.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index a7365245..d290b963 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -611,7 +611,24 @@ def process_inbox_request(request_json, store_ap_json): log_incoming_ap(request_json['id'], APLOG_ACCEPT, APLOG_SUCCESS, request_json if store_ap_json else None) return - + # Reject: remote server is rejecting our previous follow request + if request_json['type'] == 'Reject': + if request_json['object']['type'] == 'Follow': + user_ap_id = request_json['object']['actor'] + user = find_actor_or_create(user_ap_id, create_if_not_found=False) + if not user: + log_incoming_ap(request_json['id'], APLOG_ACCEPT, APLOG_FAILURE, request_json if store_ap_json else None, 'Could not find recipient of Reject') + return + join_request = CommunityJoinRequest.query.filter_by(user_id=user.id, community_id=community.id).first() + if join_request: + db.session.delete(join_request) + existing_membership = CommunityMember.query.filter_by(user_id=user.id, community_id=community.id).first() + if existing_membership: + db.session.delete(existing_membership) + cache.delete_memoized(community_membership, user, community) + db.session.commit() + log_incoming_ap(request_json['id'], APLOG_ACCEPT, APLOG_SUCCESS, request_json if store_ap_json else None) + return # -- below this point is code that will be incrementally replaced to use log_incoming_ap() instead -- From 046a15e6175daa1b69d26e45d3a0b3dabdeca223 Mon Sep 17 00:00:00 2001 From: freamon Date: Mon, 18 Nov 2024 22:05:25 +0000 Subject: [PATCH 12/36] apf part 12: Create/Update requests --- app/activitypub/routes.py | 171 +++++++++++++++++++++++++++++++++++++- app/activitypub/util.py | 72 ++++++++++++---- 2 files changed, 225 insertions(+), 18 deletions(-) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index d290b963..5dee4da9 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -25,11 +25,11 @@ from app.activitypub.util import public_key, users_total, active_half_year, acti update_post_from_activity, undo_vote, undo_downvote, post_to_page, get_redis_connection, find_reported_object, \ process_report, ensure_domains_match, can_edit, can_delete, remove_data_from_banned_user, resolve_remote_post, \ inform_followers_of_post_update, comment_model_to_json, restore_post_or_comment, ban_local_user, unban_local_user, \ - lock_post, log_incoming_ap + lock_post, log_incoming_ap, find_community_ap_id from app.utils import gibberish, get_setting, render_template, \ community_membership, ap_datetime, ip_address, can_downvote, \ can_upvote, can_create_post, awaken_dormant_instance, shorten_string, can_create_post_reply, sha256_digest, \ - community_moderators, markdown_to_html + community_moderators, markdown_to_html, html_to_text @bp.route('/testredis') @@ -630,6 +630,97 @@ def process_inbox_request(request_json, store_ap_json): log_incoming_ap(request_json['id'], APLOG_ACCEPT, APLOG_SUCCESS, request_json if store_ap_json else None) return + # Create is new content. Update is often an edit, but Updates from Lemmy can also be new content + if request_json['type'] == 'Create' or request_json['type'] == 'Update': + if request_json['object']['type'] == 'ChatMessage': + sender = user + recipient_ap_id = request_json['object']['to'][0] + recipient = find_actor_or_create(recipient_ap_id, create_if_not_found=False) + if recipient and recipient.is_local(): + if sender.created_recently() or sender.reputation <= -10: + log_incoming_ap(request_json['id'], APLOG_CHATMESSAGE, APLOG_FAILURE, request_json if store_ap_json else None, 'Sender not eligible to send') + return + elif recipient.has_blocked_user(sender.id) or recipient.has_blocked_instance(sender.instance_id): + log_incoming_ap(request_json['id'], APLOG_CHATMESSAGE, APLOG_FAILURE, request_json if store_ap_json else None, 'Sender blocked by recipient') + return + else: + # Find existing conversation to add to + existing_conversation = Conversation.find_existing_conversation(recipient=recipient, sender=sender) + if not existing_conversation: + existing_conversation = Conversation(user_id=sender.id) + existing_conversation.members.append(recipient) + existing_conversation.members.append(sender) + db.session.add(existing_conversation) + db.session.commit() + # Save ChatMessage to DB + encrypted = request_json['object']['encrypted'] if 'encrypted' in request_json['object'] else None + new_message = ChatMessage(sender_id=sender.id, recipient_id=recipient.id, conversation_id=existing_conversation.id, + body_html=request_json['object']['content'], + body=html_to_text(request_json['object']['content']), + encrypted=encrypted) + db.session.add(new_message) + existing_conversation.updated_at = utcnow() + db.session.commit() + + # Notify recipient + notify = Notification(title=shorten_string('New message from ' + sender.display_name()), + url=f'/chat/{existing_conversation.id}#message_{new_message}', user_id=recipient.id, + author_id=sender.id) + db.session.add(notify) + recipient.unread_notifications += 1 + existing_conversation.read = False + db.session.commit() + log_incoming_ap(request_json['id'], APLOG_CHATMESSAGE, APLOG_SUCCESS, request_json if store_ap_json else None) + return + # inner object of Create is not a ChatMessage + else: + if (request_json['object']['type'] == 'Note' and 'name' in request_json['object'] and # Poll Votes + 'inReplyTo' in request_json['object'] and 'attributedTo' in request_json['object']): + post_being_replied_to = Post.query.filter_by(ap_id=request_json['object']['inReplyTo']).first() + if post_being_replied_to: + poll_data = Poll.query.get(post_being_replied_to.id) + choice = PollChoice.query.filter_by(post_id=post_being_replied_to.id, choice_text=request_json['object']['name']).first() + if poll_data and choice: + poll_data.vote_for_choice(choice.id, user.id) + db.session.commit() + log_incoming_ap(request_json['id'], APLOG_CREATE, APLOG_SUCCESS, request_json if store_ap_json else None) + if post_being_replied_to.author.is_local(): + inform_followers_of_post_update(post_being_replied_to.id, user.instance_id) + return + community_ap_id = find_community_ap_id(request_json) + if not ensure_domains_match(request_json['object']): + log_incoming_ap(request_json['id'], APLOG_CREATE, APLOG_FAILURE, request_json if store_ap_json else None, 'Domains do not match') + return + community = find_actor_or_create(community_ap_id, community_only=True, create_if_not_found=False) if community_ap_id else None + if community and community.local_only: + log_incoming_ap(request_json['id'], APLOG_CREATE, APLOG_FAILURE, request_json if store_ap_json else None, 'Remote Create in local_only community') + return + if not community: + log_incoming_ap(request_json['id'], APLOG_CREATE, APLOG_FAILURE, request_json if store_ap_json else None, 'Blocked or unfound community') + return + + object_type = request_json['object']['type'] + new_content_types = ['Page', 'Article', 'Link', 'Note', 'Question'] + if object_type in new_content_types: # create or update a post + process_new_content(user, community, store_ap_json, request_json, announced=False) + return + elif object_type == 'Video': # PeerTube: editing a video (PT doesn't Announce these) + post = Post.query.filter_by(ap_id=request_json['object']['id']).first() + if post: + if user.id == post.user_id: + update_post_from_activity(post, request_json) + log_incoming_ap(request_json['id'], APLOG_UPDATE, APLOG_SUCCESS, request_json if store_ap_json else None) + return + else: + log_incoming_ap(request_json['id'], APLOG_UPDATE, APLOG_FAILURE, request_json if store_ap_json else None, 'Edit attempt denied') + return + else: + log_incoming_ap(request_json['id'], APLOG_UPDATE, APLOG_FAILURE, request_json if store_ap_json else None, 'PeerTube post not found') + return + else: + log_incoming_ap(request_json['id'], APLOG_CREATE, APLOG_FAILURE, request_json if store_ap_json else None, 'Unacceptable type (create): ' + object_type) + return + # -- below this point is code that will be incrementally replaced to use log_incoming_ap() instead -- @@ -1794,3 +1885,79 @@ def activity_result(id): return jsonify({'error': activity.result, 'message': activity.exception_message}) else: abort(404) + + +def process_new_content(user, community, store_ap_json, request_json, announced=True): + if not announced: + in_reply_to = request_json['object']['inReplyTo'] if 'inReplyTo' in request_json['object'] else None + ap_id = request_json['object']['id'] + announce_id = None + activity_json = request_json + else: + in_reply_to = request_json['object']['object']['inReplyTo'] if 'inReplyTo' in request_json['object']['object'] else None + ap_id = request_json['object']['object']['id'] + announce_id = request_json['id'] + activity_json = request_json['object'] + + if not in_reply_to: # Creating a new post + post = Post.query.filter_by(ap_id=ap_id).first() + if post: + if activity_json['type'] == 'Create': + log_incoming_ap(request_json['id'], APLOG_CREATE, APLOG_FAILURE, request_json if store_ap_json else None, 'Create processed after Update') + return + if user.id == post.user_id: + update_post_from_activity(post, activity_json) + log_incoming_ap(request_json['id'], APLOG_UPDATE, APLOG_SUCCESS, request_json if store_ap_json else None) + if not announced: + announce_activity_to_followers(post.community, post.author, request_json) + return + else: + log_incoming_ap(request_json['id'], APLOG_UPDATE, APLOG_FAILURE, request_json if store_ap_json else None, 'Edit attempt denied') + return + else: + if can_create_post(user, community): + try: + post = create_post(store_ap_json, community, activity_json, user, announce_id=announce_id) + if post: + log_incoming_ap(request_json['id'], APLOG_CREATE, APLOG_SUCCESS, request_json if store_ap_json else None) + if not announced: + announce_activity_to_followers(community, user, request_json) + return + except TypeError as e: + current_app.logger.error('TypeError: ' + str(request_json)) + log_incoming_ap(request_json['id'], APLOG_CREATE, APLOG_FAILURE, request_json if store_ap_json else None, 'TypeError. See log file.') + return + else: + log_incoming_ap(request_json['id'], APLOG_CREATE, APLOG_FAILURE, request_json if store_ap_json else None, 'User cannot create post in Community') + return + else: # Creating a reply / comment + reply = PostReply.query.filter_by(ap_id=ap_id).first() + if reply: + if activity_json['type'] == 'Create': + log_incoming_ap(request_json['id'], APLOG_CREATE, APLOG_FAILURE, request_json if store_ap_json else None, 'Create processed after Update') + return + if user.id == reply.user_id: + update_post_reply_from_activity(reply, activity_json) + log_incoming_ap(request_json['id'], APLOG_UPDATE, APLOG_SUCCESS, request_json if store_ap_json else None) + if not announced: + announce_activity_to_followers(reply.community, reply.author, request_json) + return + else: + log_incoming_ap(request_json['id'], APLOG_UPDATE, APLOG_FAILURE, request_json if store_ap_json else None, 'Edit attempt denied') + return + else: + if can_create_post_reply(user, community): + try: + reply = create_post_reply(store_ap_json, community, in_reply_to, activity_json, user, announce_id=announce_id) + if reply: + log_incoming_ap(request_json['id'], APLOG_CREATE, APLOG_SUCCESS, request_json if store_ap_json else None) + if not announced: + announce_activity_to_followers(community, user, request_json) + return + except TypeError as e: + current_app.logger.error('TypeError: ' + str(request_json)) + log_incoming_ap(request_json['id'], APLOG_CREATE, APLOG_FAILURE, request_json if store_ap_json else None, 'TypeError. See log file.') + return + else: + log_incoming_ap(request_json['id'], APLOG_CREATE, APLOG_FAILURE, request_json if store_ap_json else None, 'User cannot create reply in Community') + return diff --git a/app/activitypub/util.py b/app/activitypub/util.py index e4e92f1f..8f9bf4fd 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -1638,10 +1638,9 @@ def lock_post_task(mod_ap_id, post_id, comments_enabled): db.session.commit() -def create_post_reply(activity_log: ActivityPubLog, community: Community, in_reply_to, request_json: dict, user: User, announce_id=None) -> Union[PostReply, None]: +def create_post_reply(store_ap_json, community: Community, in_reply_to, request_json: dict, user: User, announce_id=None) -> Union[PostReply, None]: if community.local_only: - activity_log.exception_message = 'Community is local only, reply discarded' - activity_log.result = 'ignored' + log_incoming_ap(request_json['id'], APLOG_CREATE, APLOG_FAILURE, request_json if store_ap_json else None, 'Community is local only, reply discarded') return None post_id, parent_comment_id, root_id = find_reply_parent(in_reply_to) @@ -1652,7 +1651,7 @@ def create_post_reply(activity_log: ActivityPubLog, community: Community, in_rep else: parent_comment = None if post_id is None: - activity_log.exception_message = 'Could not find parent post' + log_incoming_ap(request_json['id'], APLOG_CREATE, APLOG_FAILURE, request_json if store_ap_json else None, 'Could not find parent post') return None post = Post.query.get(post_id) @@ -1678,31 +1677,29 @@ def create_post_reply(activity_log: ActivityPubLog, community: Community, in_rep language = find_language(next(iter(request_json['object']['contentMap']))) # Combination of next and iter gets the first key in a dict language_id = language.id if language else None - post_reply = None try: post_reply = PostReply.new(user, post, parent_comment, notify_author=True, body=body, body_html=body_html, language_id=language_id, request_json=request_json, announce_id=announce_id) - activity_log.result = 'success' + return post_reply except Exception as ex: - activity_log.exception_message = str(ex) - activity_log.result = 'ignored' - db.session.commit() - return post_reply + log_incoming_ap(request_json['id'], APLOG_CREATE, APLOG_FAILURE, request_json if store_ap_json else None, str(ex)) + return None + else: + log_incoming_ap(request_json['id'], APLOG_CREATE, APLOG_FAILURE, request_json if store_ap_json else None, 'Unable to find parent post/comment') + return None -def create_post(activity_log: ActivityPubLog, community: Community, request_json: dict, user: User, announce_id=None) -> Union[Post, None]: +def create_post(store_ap_json, community: Community, request_json: dict, user: User, announce_id=None) -> Union[Post, None]: if community.local_only: - activity_log.exception_message = 'Community is local only, post discarded' - activity_log.result = 'ignored' + log_incoming_ap(request_json['id'], APLOG_CREATE, APLOG_FAILURE, request_json if store_ap_json else None, 'Community is local only, post discarded') return None try: post = Post.new(user, community, request_json, announce_id) + return post except Exception as ex: - activity_log.exception_message = str(ex) + log_incoming_ap(request_json['id'], APLOG_CREATE, APLOG_FAILURE, request_json if store_ap_json else None, str(ex)) return None - return post - def notify_about_post(post: Post): # todo: eventually this function could trigger a lot of DB activity. This function will need to be a celery task. @@ -2578,3 +2575,46 @@ def log_incoming_ap(id, aplog_type, aplog_result, request_json, message=None): activity_log.activity_json = json.dumps(request_json) db.session.add(activity_log) db.session.commit() + + +def find_community_ap_id(request_json): + locations = ['audience', 'cc', 'to'] + if 'object' in request_json and isinstance(request_json['object'], dict): + rjs = [request_json, request_json['object']] + else: + rjs = [request_json] + for rj in rjs: + for location in locations: + if location in rj: + potential_id = rj[location] + if isinstance(potential_id, str): + if not potential_id.startswith('https://www.w3.org') and not potential_id.endswith('/followers'): + potential_community = Community.query.filter_by(ap_profile_id=potential_id.lower()).first() + if potential_community: + return potential_id + if isinstance(potential_id, list): + for c in potential_id: + if not c.startswith('https://www.w3.org') and not c.endswith('/followers'): + potential_community = Community.query.filter_by(ap_profile_id=c.lower()).first() + if potential_community: + return c + + if not 'object' in request_json: + return None + + if 'inReplyTo' in request_json['object'] and request_json['object']['inReplyTo'] is not None: + post_being_replied_to = Post.query.filter_by(ap_id=request_json['object']['inReplyTo']).first() + if post_being_replied_to: + return post_being_replied_to.community.ap_profile_id + else: + comment_being_replied_to = PostReply.query.filter_by(ap_id=request_json['object']['inReplyTo']).first() + if comment_being_replied_to: + return comment_being_replied_to.community.ap_profile_id + + if request_json['object']['type'] == 'Video': # PeerTube + if 'attributedTo' in request_json['object'] and isinstance(request_json['object']['attributedTo'], list): + for a in request_json['object']['attributedTo']: + if a['type'] == 'Group': + return a['id'] + + return None From 115bb8426f78f95f8bea892116d911be639de359 Mon Sep 17 00:00:00 2001 From: freamon Date: Mon, 18 Nov 2024 22:31:18 +0000 Subject: [PATCH 13/36] apf part 13: Delete requests --- app/activitypub/routes.py | 17 ++++++++ app/activitypub/util.py | 81 ++++++++++++++++----------------------- 2 files changed, 49 insertions(+), 49 deletions(-) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index 5dee4da9..c0939ec6 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -721,6 +721,23 @@ def process_inbox_request(request_json, store_ap_json): log_incoming_ap(request_json['id'], APLOG_CREATE, APLOG_FAILURE, request_json if store_ap_json else None, 'Unacceptable type (create): ' + object_type) return + if request_json['type'] == 'Delete': + if isinstance(request_json['object'], str): + ap_id = request_json['object'] # lemmy + else: + ap_id = request_json['object']['id'] # kbin + to_delete = find_liked_object(ap_id) # Just for Posts and Replies (User deletes go through process_delete_request()) + + if to_delete: + if to_delete.deleted: + log_incoming_ap(request_json['id'], APLOG_DELETE, APLOG_IGNORED, request_json if store_ap_json else None, 'Activity about local content which is already deleted') + else: + delete_post_or_comment(user, to_delete, store_ap_json, request_json) + announce_activity_to_followers(to_delete.community, user, request_json) + else: + log_incoming_ap(request_json['id'], APLOG_DELETE, APLOG_FAILURE, request_json if store_ap_json else None, 'Delete: cannot find ' + ap_id) + return + # -- below this point is code that will be incrementally replaced to use log_incoming_ap() instead -- diff --git a/app/activitypub/util.py b/app/activitypub/util.py index 8f9bf4fd..d5a35e02 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -1334,56 +1334,39 @@ def is_activitypub_request(): return 'application/ld+json' in request.headers.get('Accept', '') or 'application/activity+json' in request.headers.get('Accept', '') -def delete_post_or_comment(user_ap_id, to_be_deleted_ap_id, aplog_id): - deletor = find_actor_or_create(user_ap_id) - to_delete = find_liked_object(to_be_deleted_ap_id) - aplog = ActivityPubLog.query.get(aplog_id) - - if to_delete and to_delete.deleted: - if aplog: - aplog.result = 'ignored' - aplog.exception_message = 'Activity about local content which is already deleted' - return - - if deletor and to_delete: - community = to_delete.community - if to_delete.author.id == deletor.id or deletor.is_admin() or community.is_moderator(deletor) or community.is_instance_admin(deletor): - if isinstance(to_delete, Post): - to_delete.deleted = True - to_delete.deleted_by = deletor.id - community.post_count -= 1 - to_delete.author.post_count -= 1 - if to_delete.url and to_delete.cross_posts is not None: - old_cross_posts = Post.query.filter(Post.id.in_(to_delete.cross_posts)).all() - to_delete.cross_posts.clear() - for ocp in old_cross_posts: - if ocp.cross_posts is not None and to_delete.id in ocp.cross_posts: - ocp.cross_posts.remove(to_delete.id) - db.session.commit() - if to_delete.author.id != deletor.id: - add_to_modlog_activitypub('delete_post', deletor, community_id=community.id, - link_text=shorten_string(to_delete.title), link=f'post/{to_delete.id}') - elif isinstance(to_delete, PostReply): - to_delete.deleted = True - to_delete.deleted_by = deletor.id - to_delete.author.post_reply_count -= 1 - if not to_delete.author.bot: - to_delete.post.reply_count -= 1 - db.session.commit() - if to_delete.author.id != deletor.id: - add_to_modlog_activitypub('delete_post_reply', deletor, community_id=community.id, - link_text=f'comment on {shorten_string(to_delete.post.title)}', - link=f'post/{to_delete.post.id}#comment_{to_delete.id}') - if aplog: - aplog.result = 'success' - else: - if aplog: - aplog.result = 'failure' - aplog.exception_message = 'Deletor did not have permission' +def delete_post_or_comment(deletor, to_delete, store_ap_json, request_json): + community = to_delete.community + if to_delete.user_id == deletor.id or deletor.is_admin() or community.is_moderator(deletor) or community.is_instance_admin(deletor): + if isinstance(to_delete, Post): + to_delete.deleted = True + to_delete.deleted_by = deletor.id + community.post_count -= 1 + to_delete.author.post_count -= 1 + if to_delete.url and to_delete.cross_posts is not None: + old_cross_posts = Post.query.filter(Post.id.in_(to_delete.cross_posts)).all() + to_delete.cross_posts.clear() + for ocp in old_cross_posts: + if ocp.cross_posts is not None and to_delete.id in ocp.cross_posts: + ocp.cross_posts.remove(to_delete.id) + db.session.commit() + if to_delete.author.id != deletor.id: + add_to_modlog_activitypub('delete_post', deletor, community_id=community.id, + link_text=shorten_string(to_delete.title), link=f'post/{to_delete.id}') + elif isinstance(to_delete, PostReply): + to_delete.deleted = True + to_delete.deleted_by = deletor.id + to_delete.author.post_reply_count -= 1 + community.post_reply_count -= 1 + if not to_delete.author.bot: + to_delete.post.reply_count -= 1 + db.session.commit() + if to_delete.author.id != deletor.id: + add_to_modlog_activitypub('delete_post_reply', deletor, community_id=community.id, + link_text=f'comment on {shorten_string(to_delete.post.title)}', + link=f'post/{to_delete.post.id}#comment_{to_delete.id}') + log_incoming_ap(request_json['id'], APLOG_DELETE, APLOG_SUCCESS, request_json if store_ap_json else None) else: - if aplog: - aplog.result = 'failure' - aplog.exception_message = 'Unable to resolve deletor, or target' + log_incoming_ap(request_json['id'], APLOG_DELETE, APLOG_FAILURE, request_json if store_ap_json else None, 'Deletor did not have permisson') def restore_post_or_comment(object_json, aplog_id): From 5f960345810a875e364aee0bab54a9268f0ecb33 Mon Sep 17 00:00:00 2001 From: freamon Date: Tue, 19 Nov 2024 11:51:29 +0000 Subject: [PATCH 14/36] apf part 14: Like / Dislike requests --- app/activitypub/routes.py | 43 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index c0939ec6..3b65f3e0 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -738,6 +738,17 @@ def process_inbox_request(request_json, store_ap_json): log_incoming_ap(request_json['id'], APLOG_DELETE, APLOG_FAILURE, request_json if store_ap_json else None, 'Delete: cannot find ' + ap_id) return + if request_json['type'] == 'Like' or request_json['type'] == 'EmojiReact': # Upvote + process_upvote(user, store_ap_json, request_json, announced=False) + return + + if request_json['type'] == 'Dislike': # Downvote + if site.enable_downvotes is False: + log_incoming_ap(request_json['id'], APLOG_DISLIKE, APLOG_IGNORED, request_json if store_ap_json else None, 'Dislike ignored because of allow_dislike setting') + return + process_downvote(user, store_ap_json, request_json, announced=False) + return + # -- below this point is code that will be incrementally replaced to use log_incoming_ap() instead -- @@ -1978,3 +1989,35 @@ def process_new_content(user, community, store_ap_json, request_json, announced= else: log_incoming_ap(request_json['id'], APLOG_CREATE, APLOG_FAILURE, request_json if store_ap_json else None, 'User cannot create reply in Community') return + + +def process_upvote(user, store_ap_json, request_json, announced=True): + ap_id = request_json['object'] if not announced else request_json['object']['object'] + liked = find_liked_object(ap_id) + if liked is None: + log_incoming_ap(request_json['id'], APLOG_LIKE, APLOG_FAILURE, request_json if store_ap_json else None, 'Unfound object ' + ap_id) + return + if can_upvote(user, liked.community): + if isinstance(liked, (Post, PostReply)): + liked.vote(user, 'upvote') + log_incoming_ap(request_json['id'], APLOG_LIKE, APLOG_SUCCESS, request_json if store_ap_json else None) + if not announced: + announce_activity_to_followers(liked.community, user, request_json) + else: + log_incoming_ap(request_json['id'], APLOG_LIKE, APLOG_IGNORED, request_json if store_ap_json else None, 'Cannot upvote this') + + +def process_downvote(user, store_ap_json, request_json, announced=True): + ap_id = request_json['object'] if not announced else request_json['object']['object'] + liked = find_liked_object(ap_id) + if liked is None: + log_incoming_ap(request_json['id'], APLOG_DISLIKE, APLOG_FAILURE, request_json if store_ap_json else None, 'Unfound object ' + ap_id) + return + if can_downvote(user, liked.community): + if isinstance(liked, (Post, PostReply)): + liked.vote(user, 'downvote') + log_incoming_ap(request_json['id'], APLOG_DISLIKE, APLOG_SUCCESS, request_json if store_ap_json else None) + if not announced: + announce_activity_to_followers(liked.community, user, request_json) + else: + log_incoming_ap(request_json['id'], APLOG_DISLIKE, APLOG_IGNORED, request_json if store_ap_json else None, 'Cannot downvote this') From 9bca07504e7c1be402d95b171c4415ce04283be2 Mon Sep 17 00:00:00 2001 From: freamon Date: Tue, 19 Nov 2024 12:02:15 +0000 Subject: [PATCH 15/36] apf part 15: Flag requests --- app/activitypub/routes.py | 10 ++++++++++ app/activitypub/util.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index 3b65f3e0..fa3dd3fe 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -749,6 +749,16 @@ def process_inbox_request(request_json, store_ap_json): process_downvote(user, store_ap_json, request_json, announced=False) return + if request_json['type'] == 'Flag': # Reported content + reported = find_reported_object(request_json['object']) + if reported: + process_report(user, reported, request_json) + log_incoming_ap(request_json['id'], APLOG_REPORT, APLOG_SUCCESS, request_json if store_ap_json else None) + announce_activity_to_followers(reported.community, user, request_json) + else: + log_incoming_ap(request_json['id'], APLOG_REPORT, APLOG_IGNORED, request_json if store_ap_json else None, 'Report ignored due to missing content') + return + # -- below this point is code that will be incrementally replaced to use log_incoming_ap() instead -- diff --git a/app/activitypub/util.py b/app/activitypub/util.py index d5a35e02..a2669911 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -1987,7 +1987,7 @@ def undo_vote(activity_log, comment, post, target_ap_id, user): return None -def process_report(user, reported, request_json, activity_log): +def process_report(user, reported, request_json): if len(request_json['summary']) < 15: reasons = request_json['summary'] description = '' From dccecc15bc4657b5c491684e7bf82ff9c07740a1 Mon Sep 17 00:00:00 2001 From: freamon Date: Wed, 20 Nov 2024 19:48:38 +0000 Subject: [PATCH 16/36] apf part 16: Block requests --- app/activitypub/routes.py | 32 ++++++++++++++++++++++++++++- app/activitypub/util.py | 42 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index fa3dd3fe..dc86eeca 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -25,7 +25,7 @@ from app.activitypub.util import public_key, users_total, active_half_year, acti update_post_from_activity, undo_vote, undo_downvote, post_to_page, get_redis_connection, find_reported_object, \ process_report, ensure_domains_match, can_edit, can_delete, remove_data_from_banned_user, resolve_remote_post, \ inform_followers_of_post_update, comment_model_to_json, restore_post_or_comment, ban_local_user, unban_local_user, \ - lock_post, log_incoming_ap, find_community_ap_id + lock_post, log_incoming_ap, find_community_ap_id, site_ban_remove_data from app.utils import gibberish, get_setting, render_template, \ community_membership, ap_datetime, ip_address, can_downvote, \ can_upvote, can_create_post, awaken_dormant_instance, shorten_string, can_create_post_reply, sha256_digest, \ @@ -759,6 +759,36 @@ def process_inbox_request(request_json, store_ap_json): log_incoming_ap(request_json['id'], APLOG_REPORT, APLOG_IGNORED, request_json if store_ap_json else None, 'Report ignored due to missing content') return + if request_json['type'] == 'Block': # remote site is banning one of their users + blocker = user + blocked_ap_id = request_json['object'].lower() + blocked = User.query.filter_by(ap_profile_id=blocked_ap_id).first() + if store_ap_json: + request_json['cc'] = [] # cut very long list of instances + if not blocked: + log_incoming_ap(request_json['id'], APLOG_USERBAN, APLOG_IGNORED, request_json if store_ap_json else None, 'Does not exist here') + return + block_from_ap_id = request_json['target'] + remove_data = request_json['removeData'] if 'removeData' in request_json else False + + # Lemmy currently only sends userbans for admins banning local users + # Banning remote users is hacked by banning them from every community of which they are a part + # There's plans to change this in the future though. + if not blocker.is_instance_admin() and not blocked.instance_id == blocker.instance_id: + log_incoming_ap(request_json['id'], APLOG_USERBAN, APLOG_FAILURE, request_json if store_ap_json else None, 'Does not have permission') + return + + # request_json includes 'expires' and 'endTime' (same thing) but nowhere to record this and check in future for end in ban. + + if remove_data == True: + site_ban_remove_data(blocker.id, blocked) + log_incoming_ap(request_json['id'], APLOG_USERBAN, APLOG_SUCCESS, request_json if store_ap_json else None) + else: + #blocked.banned = True # uncommented until there's a mechanism for processing ban expiry date + #db.session.commit() + log_incoming_ap(request_json['id'], APLOG_USERBAN, APLOG_IGNORED, request_json if store_ap_json else None, 'Banned, but content retained') + return + # -- below this point is code that will be incrementally replaced to use log_incoming_ap() instead -- diff --git a/app/activitypub/util.py b/app/activitypub/util.py index a2669911..430f3e0f 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -1429,6 +1429,48 @@ def restore_post_or_comment(object_json, aplog_id): aplog.exception_message = 'Unable to resolve restorer or target' +def site_ban_remove_data(blocker_id, blocked): + replies = PostReply.query.filter_by(user_id=blocked.id, deleted=False) + for reply in replies: + reply.deleted = True + reply.deleted_by = blocker_id + if not blocked.bot: + reply.post.reply_count -= 1 + reply.community.post_reply_count -= 1 + blocked.reply_count = 0 + db.session.commit() + + posts = Post.query.filter_by(user_id=blocked.id, deleted=False) + for post in posts: + post.deleted = True + post.deleted_by = blocker_id + post.community.post_count -= 1 + if post.url and post.cross_posts is not None: + old_cross_posts = Post.query.filter(Post.id.in_(post.cross_posts)).all() + post.cross_posts.clear() + for ocp in old_cross_posts: + if ocp.cross_posts is not None and post.id in ocp.cross_posts: + ocp.cross_posts.remove(post.id) + blocked.post_count = 0 + db.session.commit() + + # Delete all their images to save moderators from having to see disgusting stuff. + # Images attached to posts can't be restored, but site ban reversals don't have a 'removeData' field anyway. + files = File.query.join(Post).filter(Post.user_id == blocked.id).all() + for file in files: + file.delete_from_disk() + file.source_url = '' + if blocked.avatar_id: + blocked.avatar.delete_from_disk() + blocked.avatar.source_url = '' + if blocked.cover_id: + blocked.cover.delete_from_disk() + blocked.cover.source_url = '' + # blocked.banned = True # uncommented until there's a mechanism for processing ban expiry date + + db.session.commit() + + def remove_data_from_banned_user(deletor_ap_id, user_ap_id, target): if current_app.debug: remove_data_from_banned_user_task(deletor_ap_id, user_ap_id, target) From 7dd8717aa7a9fd6fa66d5b4f209c5473bae0d02f Mon Sep 17 00:00:00 2001 From: freamon Date: Fri, 22 Nov 2024 02:07:22 +0000 Subject: [PATCH 17/36] apf part 17: Undo / Follow requests --- app/activitypub/routes.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index dc86eeca..cdc2e15f 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -789,6 +789,36 @@ def process_inbox_request(request_json, store_ap_json): log_incoming_ap(request_json['id'], APLOG_USERBAN, APLOG_IGNORED, request_json if store_ap_json else None, 'Banned, but content retained') return + if request_json['type'] == 'Undo': + if request_json['object']['type'] == 'Follow': # Unsubscribe from a community or user + target_ap_id = request_json['object']['object'] + target = find_actor_or_create(target_ap_id, create_if_not_found=False) + if isinstance(target, Community): + community = target + member = CommunityMember.query.filter_by(user_id=user.id, community_id=community.id).first() + join_request = CommunityJoinRequest.query.filter_by(user_id=user.id, community_id=community.id).first() + if member: + db.session.delete(member) + community.subscriptions_count -= 1 + if join_request: + db.session.delete(join_request) + db.session.commit() + cache.delete_memoized(community_membership, user, community) + log_incoming_ap(request_json['id'], APLOG_UNDO_FOLLOW, APLOG_SUCCESS, request_json if store_ap_json else None) + return + if isinstance(target, User): + local_user = target + remote_user = user + follower = UserFollower.query.filter_by(local_user_id=local_user.id, remote_user_id=remote_user.id, is_accepted=True).first() + if follower: + db.session.delete(follower) + db.session.commit() + log_incoming_ap(request_json['id'], APLOG_UNDO_FOLLOW, APLOG_SUCCESS, request_json if store_ap_json else None) + return + if not target: + log_incoming_ap(request_json['id'], APLOG_UNDO_FOLLOW, APLOG_FAILURE, request_json if store_ap_json else None, 'Unfound target') + return + # -- below this point is code that will be incrementally replaced to use log_incoming_ap() instead -- From e533603f4d9ad995673dcc891c6107943289bf6e Mon Sep 17 00:00:00 2001 From: freamon Date: Sat, 23 Nov 2024 01:29:04 +0000 Subject: [PATCH 18/36] apf part 18: Undo / Delete requests --- app/activitypub/routes.py | 22 +++++++++ app/activitypub/util.py | 93 ++++++++++++++++----------------------- 2 files changed, 59 insertions(+), 56 deletions(-) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index cdc2e15f..1630428c 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -819,6 +819,28 @@ def process_inbox_request(request_json, store_ap_json): log_incoming_ap(request_json['id'], APLOG_UNDO_FOLLOW, APLOG_FAILURE, request_json if store_ap_json else None, 'Unfound target') return + if request_json['object']['type'] == 'Delete': # Restore something previously deleted + if isinstance(request_json['object']['object'], str): + ap_id = request_json['object']['object'] # lemmy + else: + ap_id = request_json['object']['object']['id'] # kbin + if user.ap_profile_id == ap_id.lower(): + # a user is undoing a self-delete + # will never get here, because find_actor_or_create() in shared_inbox() will return None if user.deleted + return + + restorer = user + to_restore = find_liked_object(ap_id) # a user or a mod/admin is undoing the delete of a post or reply + if to_restore: + if not to_restore.deleted: + log_incoming_ap(request_json['id'], APLOG_UNDO_DELETE, APLOG_IGNORED, request_json if store_ap_json else None, 'Activity about local content which is already restored') + else: + restore_post_or_comment(restorer, to_restore, store_ap_json, request_json) + announce_activity_to_followers(to_restore.community, user, request_json) + else: + log_incoming_ap(request_json['id'], APLOG_UNDO_DELETE, APLOG_FAILURE, request_json if store_ap_json else None, 'Undo delete: cannot find ' + ap_id) + return + # -- below this point is code that will be incrementally replaced to use log_incoming_ap() instead -- diff --git a/app/activitypub/util.py b/app/activitypub/util.py index 430f3e0f..c96fe011 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -1369,64 +1369,45 @@ def delete_post_or_comment(deletor, to_delete, store_ap_json, request_json): log_incoming_ap(request_json['id'], APLOG_DELETE, APLOG_FAILURE, request_json if store_ap_json else None, 'Deletor did not have permisson') -def restore_post_or_comment(object_json, aplog_id): - restorer = find_actor_or_create(object_json['actor']) if 'actor' in object_json else None - to_restore = find_liked_object(object_json['object']) if 'object' in object_json else None - aplog = ActivityPubLog.query.get(aplog_id) +def restore_post_or_comment(restorer, to_restore, store_ap_json, request_json): + community = to_restore.community + if to_restore.user_id == restorer.id or restorer.is_admin() or community.is_moderator(restorer) or community.is_instance_admin(restorer): + if isinstance(to_restore, Post): + to_restore.deleted = False + to_restore.deleted_by = None + community.post_count += 1 + to_restore.author.post_count += 1 + if to_restore.url: + new_cross_posts = Post.query.filter(Post.id != to_restore.id, Post.url == to_restore.url, Post.deleted == False, + Post.posted_at > utcnow() - timedelta(days=6)).all() + for ncp in new_cross_posts: + if ncp.cross_posts is None: + ncp.cross_posts = [to_restore.id] + else: + ncp.cross_posts.append(to_restore.id) + if to_restore.cross_posts is None: + to_restore.cross_posts = [ncp.id] + else: + to_restore.cross_posts.append(ncp.id) + db.session.commit() + if to_restore.author.id != restorer.id: + add_to_modlog_activitypub('restore_post', restorer, community_id=community.id, + link_text=shorten_string(to_restore.title), link=f'post/{to_restore.id}') - if to_restore and not to_restore.deleted: - if aplog: - aplog.result = 'ignored' - aplog.exception_message = 'Activity about local content which is already restored' - return - - if restorer and to_restore: - community = to_restore.community - if to_restore.author.id == restorer.id or restorer.is_admin() or community.is_moderator(restorer) or community.is_instance_admin(restorer): - if isinstance(to_restore, Post): - to_restore.deleted = False - to_restore.deleted_by = None - community.post_count += 1 - to_restore.author.post_count += 1 - if to_restore.url: - new_cross_posts = Post.query.filter(Post.id != to_restore.id, Post.url == to_restore.url, Post.deleted == False, - Post.posted_at > utcnow() - timedelta(days=6)).all() - for ncp in new_cross_posts: - if ncp.cross_posts is None: - ncp.cross_posts = [to_restore.id] - else: - ncp.cross_posts.append(to_restore.id) - if to_restore.cross_posts is None: - to_restore.cross_posts = [ncp.id] - else: - to_restore.cross_posts.append(ncp.id) - db.session.commit() - if to_restore.author.id != restorer.id: - add_to_modlog_activitypub('restore_post', restorer, community_id=community.id, - link_text=shorten_string(to_restore.title), link=f'post/{to_restore.id}') - - elif isinstance(to_restore, PostReply): - to_restore.deleted = False - to_restore.deleted_by = None - if not to_restore.author.bot: - to_restore.post.reply_count += 1 - to_restore.author.post_reply_count += 1 - db.session.commit() - if to_restore.author.id != restorer.id: - add_to_modlog_activitypub('restore_post_reply', restorer, community_id=community.id, - link_text=f'comment on {shorten_string(to_restore.post.title)}', - link=f'post/{to_restore.post_id}#comment_{to_restore.id}') - - if aplog: - aplog.result = 'success' - else: - if aplog: - aplog.result = 'failure' - aplog.exception_message = 'Restorer did not have permission' + elif isinstance(to_restore, PostReply): + to_restore.deleted = False + to_restore.deleted_by = None + if not to_restore.author.bot: + to_restore.post.reply_count += 1 + to_restore.author.post_reply_count += 1 + db.session.commit() + if to_restore.author.id != restorer.id: + add_to_modlog_activitypub('restore_post_reply', restorer, community_id=community.id, + link_text=f'comment on {shorten_string(to_restore.post.title)}', + link=f'post/{to_restore.post_id}#comment_{to_restore.id}') + log_incoming_ap(request_json['id'], APLOG_UNDO_DELETE, APLOG_SUCCESS, request_json if store_ap_json else None) else: - if aplog: - aplog.result = 'failure' - aplog.exception_message = 'Unable to resolve restorer or target' + log_incoming_ap(request_json['id'], APLOG_UNDO_DELETE, APLOG_FAILURE, request_json if store_ap_json else None, 'Restorer did not have permisson') def site_ban_remove_data(blocker_id, blocked): From 30dba0acf293e8c629845083245251188c0e2f55 Mon Sep 17 00:00:00 2001 From: freamon Date: Sat, 23 Nov 2024 01:36:31 +0000 Subject: [PATCH 19/36] apf part 19: Undo / Like requests --- app/activitypub/routes.py | 11 +++++++++++ app/activitypub/util.py | 23 +++++++---------------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index 1630428c..18c193d1 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -841,6 +841,17 @@ def process_inbox_request(request_json, store_ap_json): log_incoming_ap(request_json['id'], APLOG_UNDO_DELETE, APLOG_FAILURE, request_json if store_ap_json else None, 'Undo delete: cannot find ' + ap_id) return + if request_json['object']['type'] == 'Like' or request_json['object']['type'] == 'Dislike': # Undoing an upvote or downvote + post = comment = None + target_ap_id = request_json['object']['object'] + post_or_comment = undo_vote(comment, post, target_ap_id, user) + if post_or_comment: + log_incoming_ap(request_json['id'], APLOG_UNDO_VOTE, APLOG_SUCCESS, request_json if store_ap_json else None) + announce_activity_to_followers(post_or_comment.community, user, request_json) + else: + log_incoming_ap(request_json['id'], APLOG_UNDO_VOTE, APLOG_FAILURE, request_json if store_ap_json else None, 'Unfound object ' + target_ap_id) + return + # -- below this point is code that will be incrementally replaced to use log_incoming_ap() instead -- diff --git a/app/activitypub/util.py b/app/activitypub/util.py index c96fe011..726075e9 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -1969,11 +1969,10 @@ def undo_downvote(activity_log, comment, post, target_ap_id, user): return post -def undo_vote(activity_log, comment, post, target_ap_id, user): +def undo_vote(comment, post, target_ap_id, user): voted_on = find_liked_object(target_ap_id) - if (user and not user.is_local()) and isinstance(voted_on, Post): + if isinstance(voted_on, Post): post = voted_on - user.last_seen = utcnow() existing_vote = PostVote.query.filter_by(user_id=user.id, post_id=post.id).first() if existing_vote: post.author.reputation -= existing_vote.effect @@ -1983,8 +1982,9 @@ def undo_vote(activity_log, comment, post, target_ap_id, user): post.up_votes -= 1 post.score -= existing_vote.effect db.session.delete(existing_vote) - activity_log.result = 'success' - if (user and not user.is_local()) and isinstance(voted_on, PostReply): + db.session.commit() + return post + if isinstance(voted_on, PostReply): comment = voted_on existing_vote = PostReplyVote.query.filter_by(user_id=user.id, post_reply_id=comment.id).first() if existing_vote: @@ -1995,18 +1995,9 @@ def undo_vote(activity_log, comment, post, target_ap_id, user): comment.up_votes -= 1 comment.score -= existing_vote.effect db.session.delete(existing_vote) - activity_log.result = 'success' - - if user is None or (post is None and comment is None): - activity_log.exception_message = 'Blocked or unfound user or comment' - if user and user.is_local(): - activity_log.exception_message = 'Activity about local content which is already present' - activity_log.result = 'ignored' - - if post: - return post - if comment: + db.session.commit() return comment + return None From ea15cb60796459f9b9021295bf8356e6be1c1bcf Mon Sep 17 00:00:00 2001 From: freamon Date: Sat, 23 Nov 2024 01:39:47 +0000 Subject: [PATCH 20/36] apf part 20: Undo / Block requests --- app/activitypub/routes.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index 18c193d1..9ee0f8ff 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -852,6 +852,28 @@ def process_inbox_request(request_json, store_ap_json): log_incoming_ap(request_json['id'], APLOG_UNDO_VOTE, APLOG_FAILURE, request_json if store_ap_json else None, 'Unfound object ' + target_ap_id) return + if request_json['object']['type'] == 'Block': # remote site is unbanning one of their users + unblocker = user + unblocked_ap_id = request_json['object']['object'].lower() + unblocked = User.query.filter_by(ap_profile_id=unblocked_ap_id).first() + if store_ap_json: + request_json['cc'] = [] # cut very long list of instances + request_json['object']['cc'] = [] + if not unblocked: + log_incoming_ap(request_json['id'], APLOG_USERBAN, APLOG_IGNORED, request_json if store_ap_json else None, 'Does not exist here') + return + unblock_from_ap_id = request_json['object']['target'] + + if not unblocker.is_instance_admin() and not unblocked.instance_id == blocker.instance_id: + log_incoming_ap(request_json['id'], APLOG_USERBAN, APLOG_FAILURE, request_json if store_ap_json else None, 'Does not have permission') + return + + # (no removeData field in an undo/ban - cannot restore without knowing if deletion was part of ban, or different moderator action) + #unblocked.banned = False # uncommented until there's a mechanism for processing ban expiry date + #db.session.commit() + log_incoming_ap(request_json['id'], APLOG_UNDO_USERBAN, APLOG_SUCCESS, request_json if store_ap_json else None) + return + # -- below this point is code that will be incrementally replaced to use log_incoming_ap() instead -- From a36d5b42e9e6ae97fc05bc5944fec5703e863763 Mon Sep 17 00:00:00 2001 From: freamon Date: Sat, 23 Nov 2024 19:26:25 +0000 Subject: [PATCH 21/36] apf part 21: Announce / Create or Update --- app/activitypub/routes.py | 37 +++++++++++++++++++++++++++++++++++++ app/activitypub/util.py | 12 ++++-------- 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index 9ee0f8ff..6f1cc779 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -874,6 +874,43 @@ def process_inbox_request(request_json, store_ap_json): log_incoming_ap(request_json['id'], APLOG_UNDO_USERBAN, APLOG_SUCCESS, request_json if store_ap_json else None) return + # Announce is new content and votes that happened on a remote server. + if request_json['type'] == 'Announce': + if isinstance(request_json['object'], str): # Mastodon, PeerTube, A.gup.pe + post = resolve_remote_post(request_json['object'], community.id, announce_actor=community.ap_profile_id, store_ap_json=store_ap_json) + if post: + log_incoming_ap(request_json['id'], APLOG_ANNOUNCE, APLOG_SUCCESS, request_json) + else: + log_incoming_ap(request_json['id'], APLOG_ANNOUNCE, APLOG_FAILURE, request_json, 'Could not resolve post') + return + + user_ap_id = request_json['object']['actor'] + user = find_actor_or_create(user_ap_id) + if not user or not isinstance(user, User): + log_incoming_ap(request_json['id'], APLOG_ANNOUNCE, APLOG_FAILURE, request_json, 'Blocked or unfound user for Announce object actor ' + user_ap_id) + return + + user.last_seen = site.last_active = utcnow() + user.instance.last_seen = utcnow() + user.instance.dormant = False + user.instance.gone_forever = False + user.instance.failures = 0 + db.session.commit() + + if request_json['object']['type'] == 'Create' or request_json['object']['type'] == 'Update': + object_type = request_json['object']['object']['type'] + new_content_types = ['Page', 'Article', 'Link', 'Note', 'Question'] + if object_type in new_content_types: # create or update a post + process_new_content(user, community, store_ap_json, request_json) + elif request_json['object']['type'] == 'Update' and request_json['object']['object']['type'] == 'Group': + # force refresh next time community is heard from + community.ap_fetched_at = None + db.session.commit() + log_incoming_ap(request_json['id'], APLOG_UPDATE, APLOG_SUCCESS, request_json if store_ap_json else None) + else: + log_incoming_ap(request_json['id'], APLOG_CREATE, APLOG_FAILURE, request_json if store_ap_json else None, 'Unacceptable type (create): ' + object_type) + return + # -- below this point is code that will be incrementally replaced to use log_incoming_ap() instead -- diff --git a/app/activitypub/util.py b/app/activitypub/util.py index 726075e9..f6eb5a0a 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -2291,7 +2291,7 @@ def can_delete(user_ap_id, post): return can_edit(user_ap_id, post) -def resolve_remote_post(uri: str, community_id: int, announce_actor=None) -> Union[Post, PostReply, None]: +def resolve_remote_post(uri: str, community_id: int, announce_actor=None, store_ap_json=False) -> Union[Post, PostReply, None]: post = Post.query.filter_by(ap_id=uri).first() if post: return post @@ -2371,18 +2371,14 @@ def resolve_remote_post(uri: str, community_id: int, announce_actor=None) -> Uni if not community_found: return None - activity_log = ActivityPubLog(direction='in', activity_id=post_data['id'], activity_type='Resolve Post', result='failure') - if site.log_activitypub_json: - activity_log.activity_json = json.dumps(post_data) - db.session.add(activity_log) user = find_actor_or_create(actor) if user and community and post_data: request_json = { - 'id': f"https://{uri_domain}/activities/create/gibberish(15)", + 'id': f"https://{uri_domain}/activities/create/{gibberish(15)}", 'object': post_data } if 'inReplyTo' in request_json['object'] and request_json['object']['inReplyTo']: - post_reply = create_post_reply(activity_log, community, request_json['object']['inReplyTo'], request_json, user) + post_reply = create_post_reply(store_ap_json, community, request_json['object']['inReplyTo'], request_json, user) if post_reply: if 'published' in post_data: post_reply.posted_at = post_data['published'] @@ -2391,7 +2387,7 @@ def resolve_remote_post(uri: str, community_id: int, announce_actor=None) -> Uni db.session.commit() return post_reply else: - post = create_post(activity_log, community, request_json, user) + post = create_post(store_ap_json, community, request_json, user) if post: if 'published' in post_data: post.posted_at=post_data['published'] From aad494563e772688859525fda5c88180f089dbe4 Mon Sep 17 00:00:00 2001 From: freamon Date: Sat, 23 Nov 2024 19:41:09 +0000 Subject: [PATCH 22/36] apf part 22: Announce / Delete --- app/activitypub/routes.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index 6f1cc779..b1dff5e1 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -911,6 +911,22 @@ def process_inbox_request(request_json, store_ap_json): log_incoming_ap(request_json['id'], APLOG_CREATE, APLOG_FAILURE, request_json if store_ap_json else None, 'Unacceptable type (create): ' + object_type) return + if request_json['object']['type'] == 'Delete': # Announced Delete + if isinstance(request_json['object']['object'], str): + ap_id = request_json['object']['object'] # lemmy + else: + ap_id = request_json['object']['object']['id'] # kbin + to_delete = find_liked_object(ap_id) # Just for Posts and Replies (User deletes aren't announced) + + if to_delete: + if to_delete.deleted: + log_incoming_ap(request_json['id'], APLOG_DELETE, APLOG_IGNORED, request_json if store_ap_json else None, 'Activity about local content which is already deleted') + else: + delete_post_or_comment(user, to_delete, store_ap_json, request_json) + else: + log_incoming_ap(request_json['id'], APLOG_DELETE, APLOG_FAILURE, request_json if store_ap_json else None, 'Delete: cannot find ' + ap_id) + return + # -- below this point is code that will be incrementally replaced to use log_incoming_ap() instead -- From 795250e78045f637e336a53076bfa642ce62798a Mon Sep 17 00:00:00 2001 From: freamon Date: Sat, 23 Nov 2024 19:46:27 +0000 Subject: [PATCH 23/36] apf part 23: Announce / Like or Dislike --- app/activitypub/routes.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index b1dff5e1..8f6f2b46 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -927,6 +927,17 @@ def process_inbox_request(request_json, store_ap_json): log_incoming_ap(request_json['id'], APLOG_DELETE, APLOG_FAILURE, request_json if store_ap_json else None, 'Delete: cannot find ' + ap_id) return + if request_json['object']['type'] == 'Like' or request_json['object']['type'] == 'EmojiReact': # Announced Upvote + process_upvote(user, store_ap_json, request_json) + return + + if request_json['object']['type'] == 'Dislike': # Announced Downvote + if site.enable_downvotes is False: + log_incoming_ap(request_json['id'], APLOG_DISLIKE, APLOG_IGNORED, request_json if store_ap_json else None, 'Dislike ignored because of allow_dislike setting') + return + process_downvote(user, store_ap_json, request_json) + return + # -- below this point is code that will be incrementally replaced to use log_incoming_ap() instead -- From c4665ace2102a864084a20c467ae5c426cea8b61 Mon Sep 17 00:00:00 2001 From: freamon Date: Sat, 23 Nov 2024 20:04:39 +0000 Subject: [PATCH 24/36] apf part 24: Announce / Flag --- app/activitypub/routes.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index 8f6f2b46..e4222c02 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -938,6 +938,15 @@ def process_inbox_request(request_json, store_ap_json): process_downvote(user, store_ap_json, request_json) return + if request_json['object']['type'] == 'Flag': # Announce of reported content + reported = find_reported_object(request_json['object']['object']) + if reported: + process_report(user, reported, request_json['object']) + log_incoming_ap(request_json['id'], APLOG_REPORT, APLOG_SUCCESS, request_json if store_ap_json else None) + else: + log_incoming_ap(request_json['id'], APLOG_REPORT, APLOG_IGNORED, request_json if store_ap_json else None, 'Report ignored due to missing content') + return + # -- below this point is code that will be incrementally replaced to use log_incoming_ap() instead -- From c226a6806a883194ccc2e82480b7b99c11b1f6c2 Mon Sep 17 00:00:00 2001 From: freamon Date: Sun, 24 Nov 2024 16:06:33 +0000 Subject: [PATCH 25/36] apf part 25: Announce / Lock --- app/activitypub/routes.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index e4222c02..9cdf0391 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -947,6 +947,22 @@ def process_inbox_request(request_json, store_ap_json): log_incoming_ap(request_json['id'], APLOG_REPORT, APLOG_IGNORED, request_json if store_ap_json else None, 'Report ignored due to missing content') return + if request_json['object']['type'] == 'Lock': # Announce of post lock + mod = user + post_id = request_json['object']['object'] + post = Post.query.filter_by(ap_id=post_id).first() + if post: + if post.community.is_moderator(mod) or post.community.is_instance_admin(mod): + post.comments_enabled = False + db.session.commit() + log_incoming_ap(request_json['id'], APLOG_LOCK, APLOG_SUCCESS, request_json if store_ap_json else None) + else: + log_incoming_ap(request_json['id'], APLOG_LOCK, APLOG_FAILURE, request_json if store_ap_json else None, 'Lock: Does not have permission') + else: + log_incoming_ap(request_json['id'], APLOG_LOCK, APLOG_FAILURE, request_json if store_ap_json else None, 'Lock: post not found') + return + + # -- below this point is code that will be incrementally replaced to use log_incoming_ap() instead -- From 574e3ae215f135e1cc220cf7ef3f8436d1ff8b45 Mon Sep 17 00:00:00 2001 From: freamon Date: Sun, 24 Nov 2024 16:31:20 +0000 Subject: [PATCH 26/36] apf part 26: Announce / Add mods or sticky --- app/activitypub/routes.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index 9cdf0391..a0b197e3 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -962,6 +962,36 @@ def process_inbox_request(request_json, store_ap_json): log_incoming_ap(request_json['id'], APLOG_LOCK, APLOG_FAILURE, request_json if store_ap_json else None, 'Lock: post not found') return + if request_json['object']['type'] == 'Add': # Announce of adding mods or stickying a post + target = request_json['object']['target'] + featured_url = community.ap_featured_url + moderators_url = community.ap_moderators_url + if target == featured_url: + post = Post.query.filter_by(ap_id=request_json['object']['object']).first() + if post: + post.sticky = True + db.session.commit() + log_incoming_ap(request_json['id'], APLOG_ADD, APLOG_SUCCESS, request_json if store_ap_json else None) + else: + log_incoming_ap(request_json['id'], APLOG_ADD, APLOG_FAILURE, request_json if store_ap_json else None, 'Cannot find: ' + request_json['object']['object']) + return + if target == moderators_url: + user = find_actor_or_create(request_json['object']['object']) + if user: + existing_membership = CommunityMember.query.filter_by(community_id=community.id, user_id=user.id).first() + if existing_membership: + existing_membership.is_moderator = True + else: + new_membership = CommunityMember(community_id=community.id, user_id=user.id, is_moderator=True) + db.session.add(new_membership) + db.session.commit() + log_incoming_ap(request_json['id'], APLOG_ADD, APLOG_SUCCESS, request_json if store_ap_json else None) + else: + log_incoming_ap(request_json['id'], APLOG_ADD, APLOG_FAILURE, request_json if store_ap_json else None, 'Cannot find: ' + request_json['object']['object']) + return + log_incoming_ap(request_json['id'], APLOG_ADD, APLOG_FAILURE, request_json if store_ap_json else None, 'Unknown target for Add') + return + # -- below this point is code that will be incrementally replaced to use log_incoming_ap() instead -- From 376efadd2da99779ce65f1209640f9809e7bbca8 Mon Sep 17 00:00:00 2001 From: freamon Date: Sun, 24 Nov 2024 16:35:10 +0000 Subject: [PATCH 27/36] apf part 27: Announce / Remove mods or sticky --- app/activitypub/routes.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index a0b197e3..6ed912f6 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -992,6 +992,33 @@ def process_inbox_request(request_json, store_ap_json): log_incoming_ap(request_json['id'], APLOG_ADD, APLOG_FAILURE, request_json if store_ap_json else None, 'Unknown target for Add') return + if request_json['object']['type'] == 'Remove': # Announce of removing mods or unstickying a post + target = request_json['object']['target'] + featured_url = community.ap_featured_url + moderators_url = community.ap_moderators_url + if target == featured_url: + post = Post.query.filter_by(ap_id=request_json['object']['object']).first() + if post: + post.sticky = False + db.session.commit() + log_incoming_ap(request_json['id'], APLOG_REMOVE, APLOG_SUCCESS, request_json if store_ap_json else None) + else: + log_incoming_ap(request_json['id'], APLOG_REMOVE, APLOG_FAILURE, request_json if store_ap_json else None, 'Cannot find: ' + target) + return + if target == moderators_url: + user = find_actor_or_create(request_json['object']['object'], create_if_not_found=False) + if user: + existing_membership = CommunityMember.query.filter_by(community_id=community.id, user_id=user.id).first() + if existing_membership: + existing_membership.is_moderator = False + db.session.commit() + log_incoming_ap(request_json['id'], APLOG_REMOVE, APLOG_SUCCESS, request_json if store_ap_json else None) + else: + log_incoming_ap(request_json['id'], APLOG_REMOVE, APLOG_FAILURE, request_json if store_ap_json else None, 'Cannot find: ' + request_json['object']['object']) + return + log_incoming_ap(request_json['id'], APLOG_REMOVE, APLOG_FAILURE, request_json if store_ap_json else None, 'Unknown target for Remove') + return + # -- below this point is code that will be incrementally replaced to use log_incoming_ap() instead -- From 51de9ee08269cebd05c603be01971f917f0e3a20 Mon Sep 17 00:00:00 2001 From: freamon Date: Sun, 24 Nov 2024 21:01:34 +0000 Subject: [PATCH 28/36] apf part 28: Announce / Block to ban users from communities --- app/activitypub/routes.py | 25 ++++++++- app/activitypub/util.py | 112 +++++++++++++++++++------------------- 2 files changed, 80 insertions(+), 57 deletions(-) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index 6ed912f6..dd546234 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -25,7 +25,7 @@ from app.activitypub.util import public_key, users_total, active_half_year, acti update_post_from_activity, undo_vote, undo_downvote, post_to_page, get_redis_connection, find_reported_object, \ process_report, ensure_domains_match, can_edit, can_delete, remove_data_from_banned_user, resolve_remote_post, \ inform_followers_of_post_update, comment_model_to_json, restore_post_or_comment, ban_local_user, unban_local_user, \ - lock_post, log_incoming_ap, find_community_ap_id, site_ban_remove_data + lock_post, log_incoming_ap, find_community_ap_id, site_ban_remove_data, community_ban_remove_data from app.utils import gibberish, get_setting, render_template, \ community_membership, ap_datetime, ip_address, can_downvote, \ can_upvote, can_create_post, awaken_dormant_instance, shorten_string, can_create_post_reply, sha256_digest, \ @@ -1019,6 +1019,29 @@ def process_inbox_request(request_json, store_ap_json): log_incoming_ap(request_json['id'], APLOG_REMOVE, APLOG_FAILURE, request_json if store_ap_json else None, 'Unknown target for Remove') return + if request_json['object']['type'] == 'Block': # Announce of user ban. Mod is banning a user from a community, + blocker = user # or an admin is banning a user from all the site's communities as part of a site ban + blocked_ap_id = request_json['object']['object'].lower() + blocked = User.query.filter_by(ap_profile_id=blocked_ap_id).first() + if not blocked: + log_incoming_ap(request_json['id'], APLOG_USERBAN, APLOG_IGNORED, request_json if store_ap_json else None, 'Does not exist here') + return + remove_data = request_json['object']['removeData'] if 'removeData' in request_json['object'] else False + + if not community.is_moderator(blocker) and not community.is_instance_admin(blocker): + log_incoming_ap(request_json['id'], APLOG_USERBAN, APLOG_FAILURE, request_json if store_ap_json else None, 'Does not have permission') + return + + if remove_data == True: + community_ban_remove_data(blocker.id, community.id, blocked) + log_incoming_ap(request_json['id'], APLOG_USERBAN, APLOG_SUCCESS, request_json if store_ap_json else None) + else: + log_incoming_ap(request_json['id'], APLOG_USERBAN, APLOG_IGNORED, request_json if store_ap_json else None, 'Banned, but content retained') + + if blocked.is_local(): + ban_local_user(blocker, blocked, community, request_json) + return + # -- below this point is code that will be incrementally replaced to use log_incoming_ap() instead -- diff --git a/app/activitypub/util.py b/app/activitypub/util.py index f6eb5a0a..201a3889 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -1499,76 +1499,76 @@ def remove_data_from_banned_user_task(deletor_ap_id, user_ap_id, target): db.session.commit() -def ban_local_user(deletor_ap_id, user_ap_id, target, request_json): - if current_app.debug: - ban_local_user_task(deletor_ap_id, user_ap_id, target, request_json) - else: - ban_local_user_task.delay(deletor_ap_id, user_ap_id, target, request_json) +def community_ban_remove_data(blocker_id, community_id, blocked): + replies = PostReply.query.filter_by(user_id=blocked.id, deleted=False, community_id=community_id) + for reply in replies: + reply.deleted = True + reply.deleted_by = blocker_id + if not blocked.bot: + reply.post.reply_count -= 1 + reply.community.post_reply_count -= 1 + blocked.post_reply_count -= 1 + db.session.commit() + + posts = Post.query.filter_by(user_id=blocked.id, deleted=False, community_id=community_id) + for post in posts: + post.deleted = True + post.deleted_by = blocker_id + post.community.post_count -= 1 + if post.url and post.cross_posts is not None: + old_cross_posts = Post.query.filter(Post.id.in_(post.cross_posts)).all() + post.cross_posts.clear() + for ocp in old_cross_posts: + if ocp.cross_posts is not None and post.id in ocp.cross_posts: + ocp.cross_posts.remove(post.id) + blocked.post_count -= 1 + db.session.commit() + + # Delete attached images to save moderators from having to see disgusting stuff. + files = File.query.join(Post).filter(Post.user_id == blocked.id, Post.community_id == community_id).all() + for file in files: + file.delete_from_disk() + file.source_url = '' + db.session.commit() -@celery.task -def ban_local_user_task(deletor_ap_id, user_ap_id, target, request_json): - # same info in 'Block' and 'Announce/Block' can be sent at same time, and both call this function - ban_in_progress = cache.get(f'{deletor_ap_id} is banning {user_ap_id} from {target}') - if not ban_in_progress: - cache.set(f'{deletor_ap_id} is banning {user_ap_id} from {target}', True, timeout=60) - else: - return +def ban_local_user(blocker, blocked, community, request_json): + existing = CommunityBan.query.filter_by(community_id=community.id, user_id=blocked.id).first() + if not existing: + new_ban = CommunityBan(community_id=community.id, user_id=blocked.id, banned_by=blocker.id) + if 'summary' in request_json: + new_ban.reason=request_json['object']['summary'] + if 'expires' in request_json and datetime.fromisoformat(request_json['object']['expires']) > datetime.now(timezone.utc): + new_ban.ban_until = datetime.fromisoformat(request_json['object']['expires']) + elif 'endTime' in request_json and datetime.fromisoformat(request_json['object']['endTime']) > datetime.now(timezone.utc): + new_ban.ban_until = datetime.fromisoformat(request_json['object']['endTime']) + db.session.add(new_ban) + db.session.commit() - deletor = find_actor_or_create(deletor_ap_id, create_if_not_found=False) - user = find_actor_or_create(user_ap_id, create_if_not_found=False) - community = Community.query.filter_by(ap_profile_id=target).first() - - if not deletor or not user: - return - - # site bans by admins - if deletor.instance.user_is_admin(deletor.id) and target == f"https://{deletor.instance.domain}/": - # need instance_ban table? - ... - - # community bans by mods or admins - elif community and (community.is_moderator(deletor) or community.is_instance_admin(deletor)): - existing = CommunityBan.query.filter_by(community_id=community.id, user_id=user.id).first() - - if not existing: - new_ban = CommunityBan(community_id=community.id, user_id=user.id, banned_by=deletor.id) - if 'summary' in request_json: - new_ban.reason=request_json['summary'] - - if 'expires' in request_json and datetime.fromisoformat(request_json['expires']) > datetime.now(timezone.utc): - new_ban.ban_until = datetime.fromisoformat(request_json['expires']) - elif 'endTime' in request_json and datetime.fromisoformat(request_json['endTime']) > datetime.now(timezone.utc): - new_ban.ban_until = datetime.fromisoformat(request_json['endTime']) - - db.session.add(new_ban) - db.session.commit() - - db.session.query(CommunityJoinRequest).filter(CommunityJoinRequest.community_id == community.id, CommunityJoinRequest.user_id == user.id).delete() - - community_membership_record = CommunityMember.query.filter_by(community_id=community.id, user_id=user.id).first() + db.session.query(CommunityJoinRequest).filter(CommunityJoinRequest.community_id == community.id, CommunityJoinRequest.user_id == blocked.id).delete() + community_membership_record = CommunityMember.query.filter_by(community_id=community.id, user_id=blocked.id).first() if community_membership_record: community_membership_record.is_banned = True - cache.delete_memoized(communities_banned_from, user.id) - cache.delete_memoized(joined_communities, user.id) - cache.delete_memoized(moderating_communities, user.id) - # Notify banned person notify = Notification(title=shorten_string('You have been banned from ' + community.title), - url=f'/notifications', user_id=user.id, - author_id=deletor.id) + url=f'/notifications', user_id=blocked.id, + author_id=blocker.id) db.session.add(notify) - if not current_app.debug: # user.unread_notifications += 1 hangs app if 'user' is the same person - user.unread_notifications += 1 # who pressed 'Re-submit this activity'. - db.session.commit() + if not current_app.debug: # user.unread_notifications += 1 hangs app if 'user' is the same person + blocked.unread_notifications += 1 # who pressed 'Re-submit this activity'. # Remove their notification subscription, if any db.session.query(NotificationSubscription).filter(NotificationSubscription.entity_id == community.id, - NotificationSubscription.user_id == user.id, + NotificationSubscription.user_id == blocked.id, NotificationSubscription.type == NOTIF_COMMUNITY).delete() + db.session.commit() - add_to_modlog_activitypub('ban_user', deletor, community_id=community.id, link_text=user.display_name(), link=user.link()) + cache.delete_memoized(communities_banned_from, blocked.id) + cache.delete_memoized(joined_communities, blocked.id) + cache.delete_memoized(moderating_communities, blocked.id) + + add_to_modlog_activitypub('ban_user', blocker, community_id=community.id, link_text=blocked.display_name(), link=blocked.link()) def unban_local_user(deletor_ap_id, user_ap_id, target): From 4c4fddba8031bf018feceabd27952d84fde22fb0 Mon Sep 17 00:00:00 2001 From: freamon Date: Sun, 24 Nov 2024 21:10:16 +0000 Subject: [PATCH 29/36] apf part 29: Announce / Undo / Delete --- app/activitypub/routes.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index dd546234..c0982d7c 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -1042,6 +1042,24 @@ def process_inbox_request(request_json, store_ap_json): ban_local_user(blocker, blocked, community, request_json) return + if request_json['object']['type'] == 'Undo': + if request_json['object']['object']['type'] == 'Delete': # Announce of undo of Delete + if isinstance(request_json['object']['object']['object'], str): + ap_id = request_json['object']['object']['object'] # lemmy + else: + ap_id = request_json['object']['object']['object']['id'] # kbin + + restorer = user + to_restore = find_liked_object(ap_id) # a user or a mod/admin is undoing the delete of a post or reply + if to_restore: + if not to_restore.deleted: + log_incoming_ap(request_json['id'], APLOG_UNDO_DELETE, APLOG_IGNORED, request_json if store_ap_json else None, 'Content was not deleted') + else: + restore_post_or_comment(restorer, to_restore, store_ap_json, request_json) + else: + log_incoming_ap(request_json['id'], APLOG_UNDO_DELETE, APLOG_FAILURE, request_json if store_ap_json else None, 'Undo delete: cannot find ' + ap_id) + return + # -- below this point is code that will be incrementally replaced to use log_incoming_ap() instead -- From 381c4a88373ca3880dadbe0741a60a24b1f025ce Mon Sep 17 00:00:00 2001 From: freamon Date: Sun, 24 Nov 2024 21:19:07 +0000 Subject: [PATCH 30/36] apf part 30: Announce / Undo / Like or Dislike --- app/activitypub/routes.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index c0982d7c..1db3c44b 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -1060,6 +1060,16 @@ def process_inbox_request(request_json, store_ap_json): log_incoming_ap(request_json['id'], APLOG_UNDO_DELETE, APLOG_FAILURE, request_json if store_ap_json else None, 'Undo delete: cannot find ' + ap_id) return + if request_json['object']['object']['type'] == 'Like' or request_json['object']['object']['type'] == 'Dislike': # Announce of undo of upvote or downvote + post = comment = None + target_ap_id = request_json['object']['object']['object'] + post_or_comment = undo_vote(comment, post, target_ap_id, user) + if post_or_comment: + log_incoming_ap(request_json['id'], APLOG_UNDO_VOTE, APLOG_SUCCESS, request_json if store_ap_json else None) + else: + log_incoming_ap(request_json['id'], APLOG_UNDO_VOTE, APLOG_FAILURE, request_json if store_ap_json else None, 'Unfound object ' + target_ap_id) + return + # -- below this point is code that will be incrementally replaced to use log_incoming_ap() instead -- From 70638c39d1145891c06aa32566d75289573a3373 Mon Sep 17 00:00:00 2001 From: freamon Date: Sun, 24 Nov 2024 21:30:41 +0000 Subject: [PATCH 31/36] apf part 31: Announce / Undo / Lock --- app/activitypub/routes.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index 1db3c44b..d83b0759 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -1070,6 +1070,20 @@ def process_inbox_request(request_json, store_ap_json): log_incoming_ap(request_json['id'], APLOG_UNDO_VOTE, APLOG_FAILURE, request_json if store_ap_json else None, 'Unfound object ' + target_ap_id) return + if request_json['object']['object']['type'] == 'Lock': # Announce of undo of post lock + mod = user + post_id = request_json['object']['object']['object'] + post = Post.query.filter_by(ap_id=post_id).first() + if post: + if post.community.is_moderator(mod) or post.community.is_instance_admin(mod): + post.comments_enabled = True + db.session.commit() + log_incoming_ap(request_json['id'], APLOG_LOCK, APLOG_SUCCESS, request_json if store_ap_json else None) + else: + log_incoming_ap(request_json['id'], APLOG_LOCK, APLOG_FAILURE, request_json if store_ap_json else None, 'Lock: Does not have permission') + else: + log_incoming_ap(request_json['id'], APLOG_LOCK, APLOG_FAILURE, request_json if store_ap_json else None, 'Lock: post not found') + return # -- below this point is code that will be incrementally replaced to use log_incoming_ap() instead -- From 175094496a6fb86fc66a7f5310a63affc4a62840 Mon Sep 17 00:00:00 2001 From: freamon Date: Sun, 24 Nov 2024 21:43:08 +0000 Subject: [PATCH 32/36] apf part 32: Announce / Undo / Block --- app/activitypub/routes.py | 18 +++++++++++ app/activitypub/util.py | 66 ++++++++++----------------------------- 2 files changed, 34 insertions(+), 50 deletions(-) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index d83b0759..dc2add51 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -1085,6 +1085,24 @@ def process_inbox_request(request_json, store_ap_json): log_incoming_ap(request_json['id'], APLOG_LOCK, APLOG_FAILURE, request_json if store_ap_json else None, 'Lock: post not found') return + if request_json['object']['object']['type'] == 'Block': # Announce of undo of user ban. Mod is unbanning a user from a community, + blocker = user # or an admin is unbanning a user from all the site's communities as part of a site unban + blocked_ap_id = request_json['object']['object']['object'].lower() + blocked = User.query.filter_by(ap_profile_id=blocked_ap_id).first() + if not blocked: + log_incoming_ap(request_json['id'], APLOG_USERBAN, APLOG_IGNORED, request_json if store_ap_json else None, 'Does not exist here') + return + + if not community.is_moderator(blocker) and not community.is_instance_admin(blocker): + log_incoming_ap(request_json['id'], APLOG_USERBAN, APLOG_FAILURE, request_json if store_ap_json else None, 'Does not have permission') + return + + if blocked.is_local(): + unban_local_user(blocker, blocked, community, request_json) + log_incoming_ap(request_json['id'], APLOG_USERBAN, APLOG_SUCCESS, request_json if store_ap_json else None) + + return + # -- below this point is code that will be incrementally replaced to use log_incoming_ap() instead -- diff --git a/app/activitypub/util.py b/app/activitypub/util.py index 201a3889..d36f0b70 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -1571,60 +1571,26 @@ def ban_local_user(blocker, blocked, community, request_json): add_to_modlog_activitypub('ban_user', blocker, community_id=community.id, link_text=blocked.display_name(), link=blocked.link()) -def unban_local_user(deletor_ap_id, user_ap_id, target): - if current_app.debug: - unban_local_user_task(deletor_ap_id, user_ap_id, target) - else: - unban_local_user_task.delay(deletor_ap_id, user_ap_id, target) +def unban_local_user(blocker, blocked, community, request_json): + db.session.query(CommunityBan).filter(CommunityBan.community_id == community.id, CommunityBan.user_id == blocked.id).delete() + community_membership_record = CommunityMember.query.filter_by(community_id=community.id, user_id=blocked.id).first() + if community_membership_record: + community_membership_record.is_banned = False + # Notify unbanned person + notify = Notification(title=shorten_string('You have been unbanned from ' + community.title), + url=f'/notifications', user_id=blocked.id, author_id=blocker.id) + db.session.add(notify) + if not current_app.debug: # user.unread_notifications += 1 hangs app if 'user' is the same person + blocked.unread_notifications += 1 # who pressed 'Re-submit this activity'. -@celery.task -def unban_local_user_task(deletor_ap_id, user_ap_id, target): - # same info in 'Block' and 'Announce/Block' can be sent at same time, and both call this function - unban_in_progress = cache.get(f'{deletor_ap_id} is undoing ban of {user_ap_id} from {target}') - if not unban_in_progress: - cache.set(f'{deletor_ap_id} is undoing ban of {user_ap_id} from {target}', True, timeout=60) - else: - return + db.session.commit() - deletor = find_actor_or_create(deletor_ap_id, create_if_not_found=False) - user = find_actor_or_create(user_ap_id, create_if_not_found=False) - community = Community.query.filter_by(ap_profile_id=target).first() + cache.delete_memoized(communities_banned_from, blocked.id) + cache.delete_memoized(joined_communities, blocked.id) + cache.delete_memoized(moderating_communities, blocked.id) - if not deletor or not user: - return - - # site undo bans by admins - if deletor.instance.user_is_admin(deletor.id) and target == f"https://{deletor.instance.domain}/": - # need instance_ban table? - ... - - # community undo bans by mods or admins - elif community and (community.is_moderator(deletor) or community.is_instance_admin(deletor)): - existing_ban = CommunityBan.query.filter_by(community_id=community.id, user_id=user.id).first() - if existing_ban: - db.session.delete(existing_ban) - db.session.commit() - - community_membership_record = CommunityMember.query.filter_by(community_id=community.id, user_id=user.id).first() - if community_membership_record: - community_membership_record.is_banned = False - db.session.commit() - - cache.delete_memoized(communities_banned_from, user.id) - cache.delete_memoized(joined_communities, user.id) - cache.delete_memoized(moderating_communities, user.id) - - # Notify previously banned person - notify = Notification(title=shorten_string('You have been un-banned from ' + community.title), - url=f'/notifications', user_id=user.id, - author_id=deletor.id) - db.session.add(notify) - if not current_app.debug: # user.unread_notifications += 1 hangs app if 'user' is the same person - user.unread_notifications += 1 # who pressed 'Re-submit this activity'. - db.session.commit() - - add_to_modlog_activitypub('unban_user', deletor, community_id=community.id, link_text=user.display_name(), link=user.link()) + add_to_modlog_activitypub('unban_user', blocker, community_id=community.id, link_text=blocked.display_name(), link=blocked.link()) def lock_post(mod_ap_id, post_id, comments_enabled): From 3d4ea6637cabcdc392039f997aa9a968a7e4451e Mon Sep 17 00:00:00 2001 From: freamon Date: Sun, 24 Nov 2024 21:45:39 +0000 Subject: [PATCH 33/36] apf part 33: Log unmatched activity, and delete old code --- app/activitypub/routes.py | 802 +------------------------------------- 1 file changed, 1 insertion(+), 801 deletions(-) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index dc2add51..5b5e0c5f 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -1103,807 +1103,7 @@ def process_inbox_request(request_json, store_ap_json): return - - # -- below this point is code that will be incrementally replaced to use log_incoming_ap() instead -- - - # save all incoming data to aid in debugging and development. Set result to 'success' if things go well - activity_log = ActivityPubLog(direction='in', result='failure') - - if 'id' in request_json: - activity_log.activity_id = request_json['id'] - if site.log_activitypub_json: - activity_log.activity_json = json.dumps(request_json) - activity_log.result = 'processing' - db.session.add(activity_log) - db.session.commit() - - if 'type' in request_json: - activity_log.activity_type = request_json['type'] - if not instance_blocked(request_json['id']): - # Create is new content. Update is often an edit, but Updates from Lemmy can also be new content - if request_json['type'] == 'Create' or request_json['type'] == 'Update': - activity_log.activity_type = 'Create' - user_ap_id = request_json['object']['attributedTo'] if 'attributedTo' in request_json['object'] and isinstance(request_json['object']['attributedTo'], str) else None - if user_ap_id is None: # if there is no attributedTo, fall back to the actor on the parent object - user_ap_id = request_json['actor'] if 'actor' in request_json and isinstance(request_json['actor'], str) else None - if request_json['object']['type'] == 'ChatMessage': - activity_log.activity_type = 'Create ChatMessage' - sender = find_actor_or_create(user_ap_id) - recipient_ap_id = request_json['object']['to'][0] - recipient = find_actor_or_create(recipient_ap_id) - if sender and recipient and recipient.is_local(): - if sender.created_recently() or sender.reputation <= -10: - activity_log.exception_message = "Sender not eligible to send" - elif recipient.has_blocked_user(sender.id) or recipient.has_blocked_instance(sender.instance_id): - activity_log.exception_message = "Sender blocked by recipient" - else: - # Find existing conversation to add to - existing_conversation = Conversation.find_existing_conversation(recipient=recipient, sender=sender) - if not existing_conversation: - existing_conversation = Conversation(user_id=sender.id) - existing_conversation.members.append(recipient) - existing_conversation.members.append(sender) - db.session.add(existing_conversation) - db.session.commit() - # Save ChatMessage to DB - encrypted = request_json['object']['encrypted'] if 'encrypted' in request_json['object'] else None - new_message = ChatMessage(sender_id=sender.id, recipient_id=recipient.id, conversation_id=existing_conversation.id, - body=request_json['object']['source']['content'], - body_html=markdown_to_html(request_json['object']['source']['content']), - encrypted=encrypted) - db.session.add(new_message) - existing_conversation.updated_at = utcnow() - db.session.commit() - - # Notify recipient - notify = Notification(title=shorten_string('New message from ' + sender.display_name()), - url=f'/chat/{existing_conversation.id}#message_{new_message}', user_id=recipient.id, - author_id=sender.id) - db.session.add(notify) - recipient.unread_notifications += 1 - existing_conversation.read = False - db.session.commit() - activity_log.result = 'success' - else: - try: - community_ap_id = '' - locations = ['audience', 'cc', 'to'] - if 'object' in request_json: - rjs = [request_json, request_json['object']] - else: - rjs = [request_json] - followers_suffix = '/followers' - for rj in rjs: - for location in locations: - if location in rj: - potential_id = rj[location] - if isinstance(potential_id, str): - if not potential_id.startswith('https://www.w3.org') and not potential_id.endswith(followers_suffix): - community_ap_id = potential_id - if isinstance(potential_id, list): - for c in potential_id: - if not c.startswith('https://www.w3.org') and not c.endswith(followers_suffix): - community_ap_id = c - break - if community_ap_id: - break - if community_ap_id: - break - if not community_ap_id and 'object' in request_json and \ - 'inReplyTo' in request_json['object'] and request_json['object']['inReplyTo'] is not None: - post_being_replied_to = Post.query.filter_by(ap_id=request_json['object']['inReplyTo']).first() - if post_being_replied_to: - community_ap_id = post_being_replied_to.community.ap_profile_id - else: - comment_being_replied_to = PostReply.query.filter_by(ap_id=request_json['object']['inReplyTo']).first() - if comment_being_replied_to: - community_ap_id = comment_being_replied_to.community.ap_profile_id - if not community_ap_id and 'object' in request_json and request_json['object']['type'] == 'Video': # PeerTube - if 'attributedTo' in request_json['object'] and isinstance(request_json['object']['attributedTo'], list): - for a in request_json['object']['attributedTo']: - if a['type'] == 'Group': - community_ap_id = a['id'] - if a['type'] == 'Person': - user_ap_id = a['id'] - if not community_ap_id: - activity_log.result = 'failure' - activity_log.exception_message = 'Unable to extract community' - db.session.commit() - return - except: - activity_log.activity_type = 'exception' - db.session.commit() - return - if 'object' in request_json: - if not ensure_domains_match(request_json['object']): - activity_log.result = 'failure' - activity_log.exception_message = 'Domains do not match' - db.session.commit() - return - community = find_actor_or_create(community_ap_id, community_only=True) - if community and community.local_only: - activity_log.exception_message = 'Remote Create in local_only community' - activity_log.result = 'ignored' - db.session.commit() - return - user = find_actor_or_create(user_ap_id) - if user and not user.is_local(): - if community: - user.last_seen = community.last_active = site.last_active = utcnow() - else: - user.last_seen = site.last_active = utcnow() - object_type = request_json['object']['type'] - new_content_types = ['Page', 'Article', 'Link', 'Note', 'Question'] - if object_type in new_content_types: # create or update a post - in_reply_to = request_json['object']['inReplyTo'] if 'inReplyTo' in request_json['object'] else None - if not in_reply_to: # Creating a new post - post = Post.query.filter_by(ap_id=request_json['object']['id']).first() - if post: - if request_json['type'] == 'Create': - activity_log.result = 'ignored' - activity_log.exception_message = 'Create received for already known object' - db.session.commit() - return - else: - activity_log.activity_type = 'Update' - if can_edit(request_json['actor'], post): - update_post_from_activity(post, request_json) - announce_activity_to_followers(post.community, post.author, request_json) - activity_log.result = 'success' - else: - activity_log.exception_message = 'Edit attempt denied' - else: - if can_create_post(user, community): - try: - post = create_post(activity_log, community, request_json, user) - if post: - announce_activity_to_followers(community, user, request_json) - activity_log.result = 'success' - except TypeError as e: - activity_log.exception_message = 'TypeError. See log file.' - current_app.logger.error('TypeError: ' + str(request_json)) - post = None - else: - post = None - else: # Creating a reply / comment - reply = PostReply.query.filter_by(ap_id=request_json['object']['id']).first() - if reply: - if request_json['type'] == 'Create': - activity_log.result = 'ignored' - activity_log.exception_message = 'Create received for already known object' - db.session.commit() - return - else: - activity_log.activity_type = 'Update' - if can_edit(request_json['actor'], reply): - update_post_reply_from_activity(reply, request_json) - announce_activity_to_followers(reply.community, reply.author, request_json) - activity_log.result = 'success' - else: - activity_log.exception_message = 'Edit attempt denied' - else: - if community is None: # Mastodon: replies do not specify the community they're in. Attempt to find out the community by looking at the parent object - parent_post_id, parent_comment_id, _ = find_reply_parent(in_reply_to) - if parent_comment_id: - community = PostReply.query.get(parent_comment_id).community - elif parent_post_id: - community = Post.query.get(parent_post_id).community - if can_create_post_reply(user, community): - try: - post_reply = create_post_reply(activity_log, community, in_reply_to, request_json, user) - if post_reply: - announce_activity_to_followers(community, user, request_json) - except TypeError as e: - activity_log.exception_message = 'TypeError. See log file.' - current_app.logger.error('TypeError: ' + str(request_json)) - post = None - else: - post = None - elif object_type == 'Video': # PeerTube: editing a video (PT doesn't seem to Announce these) - post = Post.query.filter_by(ap_id=request_json['object']['id']).first() - activity_log.activity_type = 'Update' - if post: - if can_edit(request_json['actor'], post): - update_post_from_activity(post, request_json) - activity_log.result = 'success' - else: - activity_log.exception_message = 'Edit attempt denied' - else: - activity_log.exception_message = 'Post not found' - else: - activity_log.exception_message = 'Unacceptable type (create): ' + object_type - else: - if user is None or community is None: - activity_log.exception_message = 'Blocked or unfound user or community' - if user and user.is_local(): - activity_log.exception_message = 'Activity about local content which is already present' - activity_log.result = 'ignored' - - # Announce is new content and votes that happened on a remote server. - if request_json['type'] == 'Announce': - if isinstance(request_json['object'], str): # Mastodon, PeerTube, A.gup.pe - activity_log.activity_json = json.dumps(request_json) - activity_log.exception_message = 'invalid json?' - if 'actor' in request_json: - community = find_actor_or_create(request_json['actor'], community_only=True, create_if_not_found=False) - if community: - post = resolve_remote_post(request_json['object'], community.id, request_json['actor']) - elif request_json['object']['type'] == 'Create' or request_json['object']['type'] == 'Update': - activity_log.activity_type = request_json['object']['type'] - if 'object' in request_json and 'object' in request_json['object']: - if not ensure_domains_match(request_json['object']['object']): - activity_log.exception_message = 'Domains do not match' - activity_log.result = 'failure' - db.session.commit() - return - user_ap_id = request_json['object']['object']['attributedTo'] - try: - community_ap_id = request_json['object']['audience'] if 'audience' in request_json['object'] else request_json['actor'] - except KeyError: - activity_log.activity_type = 'exception' - db.session.commit() - return - community = find_actor_or_create(community_ap_id, community_only=True) - user = find_actor_or_create(user_ap_id) - if (user and not user.is_local()) and community: - user.last_seen = community.last_active = site.last_active = utcnow() - object_type = request_json['object']['object']['type'] - new_content_types = ['Page', 'Article', 'Link', 'Note'] - if object_type in new_content_types: # create a new post - in_reply_to = request_json['object']['object']['inReplyTo'] if 'inReplyTo' in \ - request_json['object']['object'] else None - if not in_reply_to: - post = Post.query.filter_by(ap_id=request_json['object']['object']['id']).first() - if post: - if request_json['object']['type'] == 'Create': - activity_log.result = 'ignored' - activity_log.exception_message = 'Create received for already known object' - db.session.commit() - return - else: - try: - update_post_from_activity(post, request_json['object']) - except KeyError: - activity_log.result = 'exception' - db.session.commit() - return - activity_log.result = 'success' - else: # activity was a Create, or an Update sent instead of a Create - if can_create_post(user, community): - post = create_post(activity_log, community, request_json['object'], user, announce_id=request_json['id']) - else: - post = None - else: - reply = PostReply.query.filter_by(ap_id=request_json['object']['object']['id']).first() - if reply: - if request_json['object']['type'] == 'Create': - activity_log.result = 'ignored' - activity_log.exception_message = 'Create received for already known object' - db.session.commit() - return - else: - try: - update_post_reply_from_activity(reply, request_json['object']) - except KeyError: - activity_log.result = 'exception' - db.session.commit() - return - activity_log.result = 'success' - else: # activity was a Create, or an Update sent instead of a Create - if can_create_post_reply(user, community): - post = create_post_reply(activity_log, community, in_reply_to, request_json['object'], user, announce_id=request_json['id']) - else: - post = None - else: - activity_log.exception_message = 'Unacceptable type: ' + object_type - else: - if user is None or community is None: - activity_log.exception_message = 'Blocked or unfound user or community' - if user and user.is_local(): - activity_log.exception_message = 'Activity about local content which is already present' - activity_log.result = 'ignored' - - elif request_json['object']['type'] == 'Like' or request_json['object']['type'] == 'EmojiReact': - activity_log.activity_type = request_json['object']['type'] - user_ap_id = request_json['object']['actor'] - liked_ap_id = request_json['object']['object'] - user = find_actor_or_create(user_ap_id) - liked = find_liked_object(liked_ap_id) - - if user is None: - activity_log.exception_message = 'Blocked or unfound user' - elif liked is None: - activity_log.exception_message = 'Unfound object ' + liked_ap_id - elif user.is_local(): - activity_log.exception_message = 'Activity about local content which is already present' - activity_log.result = 'ignored' - elif can_upvote(user, liked.community): - # insert into voted table - if liked is None: - activity_log.exception_message = 'Liked object not found' - elif liked is not None and isinstance(liked, (Post, PostReply)): - liked.vote(user, 'upvote') - activity_log.result = 'success' - else: - activity_log.exception_message = 'Could not detect type of like' - else: - activity_log.exception_message = 'Cannot upvote this' - activity_log.result = 'ignored' - - elif request_json['object']['type'] == 'Dislike': - activity_log.activity_type = request_json['object']['type'] - if site.enable_downvotes is False: - activity_log.exception_message = 'Dislike ignored because of allow_dislike setting' - else: - user_ap_id = request_json['object']['actor'] - liked_ap_id = request_json['object']['object'] - user = find_actor_or_create(user_ap_id) - disliked = find_liked_object(liked_ap_id) - if user is None: - activity_log.exception_message = 'Blocked or unfound user' - elif disliked is None: - activity_log.exception_message = 'Unfound object ' + liked_ap_id - elif user.is_local(): - activity_log.exception_message = 'Activity about local content which is already present' - activity_log.result = 'ignored' - elif can_downvote(user, disliked.community, site): - # insert into voted table - if disliked is None: - activity_log.exception_message = 'Liked object not found' - elif isinstance(disliked, (Post, PostReply)): - disliked.vote(user, 'downvote') - activity_log.result = 'success' - # todo: recalculate 'hotness' of liked post/reply - else: - activity_log.exception_message = 'Could not detect type of like' - else: - activity_log.exception_message = 'Cannot downvote this' - activity_log.result = 'ignored' - elif request_json['object']['type'] == 'Delete': - activity_log.activity_type = request_json['object']['type'] - user_ap_id = request_json['object']['actor'] - to_be_deleted_ap_id = request_json['object']['object'] - if isinstance(to_be_deleted_ap_id, dict): - activity_log.result = 'failure' - activity_log.exception_message = 'dict instead of string ' + str(to_be_deleted_ap_id) - else: - delete_post_or_comment(user_ap_id, to_be_deleted_ap_id, activity_log.id) - elif request_json['object']['type'] == 'Page': # Sent for Mastodon's benefit - activity_log.result = 'ignored' - activity_log.exception_message = 'Intended for Mastodon' - elif request_json['object']['type'] == 'Note': # Never sent? - activity_log.result = 'ignored' - activity_log.exception_message = 'Intended for Mastodon' - elif request_json['object']['type'] == 'Undo': - if request_json['object']['object']['type'] == 'Like' or request_json['object']['object']['type'] == 'Dislike': - activity_log.activity_type = request_json['object']['object']['type'] - user_ap_id = request_json['object']['actor'] - user = find_actor_or_create(user_ap_id) - post = None - comment = None - target_ap_id = request_json['object']['object']['object'] # object object object! - post = undo_vote(activity_log, comment, post, target_ap_id, user) - elif request_json['object']['object']['type'] == 'Delete': - if 'object' in request_json and 'object' in request_json['object']: - restore_post_or_comment(request_json['object']['object'], activity_log.id) - elif request_json['object']['object']['type'] == 'Block': - activity_log.activity_type = 'Undo User Ban' - deletor_ap_id = request_json['object']['object']['actor'] - user_ap_id = request_json['object']['object']['object'] - target = request_json['object']['object']['target'] - if target == request_json['actor'] and user_ap_id.startswith('https://' + current_app.config['SERVER_NAME']): - unban_local_user(deletor_ap_id, user_ap_id, target) - activity_log.result = 'success' - elif request_json['object']['object']['type'] == 'Lock' and 'object' in request_json['object']['object']: - activity_log.activity_type = 'Post Unlock' - mod_ap_id = request_json['object']['object']['actor'] - post_id = request_json['object']['object']['object'] - lock_post(mod_ap_id, post_id, True) - activity_log.result = 'success' - elif request_json['object']['type'] == 'Add' and 'target' in request_json['object']: - activity_log.activity_type = request_json['object']['type'] - target = request_json['object']['target'] - community = Community.query.filter_by(ap_public_url=request_json['actor']).first() - if community: - featured_url = community.ap_featured_url - moderators_url = community.ap_moderators_url - if target == featured_url: - post = Post.query.filter_by(ap_id=request_json['object']['object']).first() - if post: - post.sticky = True - activity_log.result = 'success' - if target == moderators_url: - user = find_actor_or_create(request_json['object']['object']) - if user: - existing_membership = CommunityMember.query.filter_by(community_id=community.id, user_id=user.id).first() - if existing_membership: - existing_membership.is_moderator = True - else: - new_membership = CommunityMember(community_id=community.id, user_id=user.id, is_moderator=True) - db.session.add(new_membership) - db.session.commit() - activity_log.result = 'success' - elif request_json['object']['type'] == 'Remove' and 'target' in request_json['object']: - activity_log.activity_type = request_json['object']['type'] - target = request_json['object']['target'] - community = Community.query.filter_by(ap_public_url=request_json['actor']).first() - if community: - featured_url = community.ap_featured_url - moderators_url = community.ap_moderators_url - if target == featured_url: - post = Post.query.filter_by(ap_id=request_json['object']['object']).first() - if post: - post.sticky = False - activity_log.result = 'success' - if target == moderators_url: - user = find_actor_or_create(request_json['object']['object'], create_if_not_found=False) - if user: - existing_membership = CommunityMember.query.filter_by(community_id=community.id, user_id=user.id).first() - if existing_membership: - existing_membership.is_moderator = False - activity_log.result = 'success' - elif request_json['object']['type'] == 'Block' and 'target' in request_json['object']: - activity_log.activity_type = 'User Ban' - deletor_ap_id = request_json['object']['actor'] - user_ap_id = request_json['object']['object'] - target = request_json['object']['target'] - remove_data = request_json['object']['removeData'] - if target == request_json['actor']: - if remove_data == True: - remove_data_from_banned_user(deletor_ap_id, user_ap_id, target) - if user_ap_id.startswith('https://' + current_app.config['SERVER_NAME']): - ban_local_user(deletor_ap_id, user_ap_id, target, request_json['object']) - activity_log.result = 'success' - elif request_json['object']['type'] == 'Lock' and 'object' in request_json['object']: - activity_log.activity_type = 'Post Lock' - mod_ap_id = request_json['object']['actor'] - post_id = request_json['object']['object'] - lock_post(mod_ap_id, post_id, False) - activity_log.result = 'success' - else: - activity_log.exception_message = 'Invalid type for Announce' - - # Follow: remote user wants to join/follow one of our communities - elif request_json['type'] == 'Follow': # Follow is when someone wants to join a community - user_ap_id = request_json['actor'] - community_ap_id = request_json['object'] - follow_id = request_json['id'] - user = find_actor_or_create(user_ap_id) - community = find_actor_or_create(community_ap_id, community_only=True) - if isinstance(community, Community): - if community and community.local_only and user: - activity_log.exception_message = 'Local only cannot be followed by remote users' - - # send reject message to deny the follow - reject = { - "@context": default_context(), - "actor": community.public_url(), - "to": [ - user.public_url() - ], - "object": { - "actor": user.public_url(), - "to": None, - "object": community.public_url(), - "type": "Follow", - "id": follow_id - }, - "type": "Reject", - "id": f"https://{current_app.config['SERVER_NAME']}/activities/reject/" + gibberish(32) - } - # Lemmy doesn't yet understand Reject/Follow, so send without worrying about response for now. - post_request(user.ap_inbox_url, reject, community.private_key, f"{community.public_url()}#main-key") - else: - if user is not None and community is not None: - # check if user is banned from this community - banned = CommunityBan.query.filter_by(user_id=user.id, community_id=community.id).first() - if banned is None: - user.last_seen = utcnow() - if community_membership(user, community) != SUBSCRIPTION_MEMBER: - member = CommunityMember(user_id=user.id, community_id=community.id) - db.session.add(member) - db.session.commit() - cache.delete_memoized(community_membership, user, community) - # send accept message to acknowledge the follow - accept = { - "@context": default_context(), - "actor": community.public_url(), - "to": [ - user.public_url() - ], - "object": { - "actor": user.public_url(), - "to": None, - "object": community.public_url(), - "type": "Follow", - "id": follow_id - }, - "type": "Accept", - "id": f"https://{current_app.config['SERVER_NAME']}/activities/accept/" + gibberish(32) - } - if post_request(user.ap_inbox_url, accept, community.private_key, f"{community.public_url()}#main-key") is True: - activity_log.result = 'success' - else: - activity_log.exception_message = 'Error sending Accept' - else: - activity_log.exception_message = 'user is banned from this community' - elif isinstance(community, User): # Pixelfed sends follow requests to the shared inbox, not the user inbox... - if current_app.debug: - process_user_follow_request(request_json, activity_log.id, user.id) - else: - process_user_follow_request.delay(request_json, activity_log.id, user.id) - # Accept: remote server is accepting our previous follow request - elif request_json['type'] == 'Accept': - if isinstance(request_json['object'], str): # a.gup.pe accepts using a string with the ID of the follow request - join_request_parts = request_json['object'].split('/') - join_request = CommunityJoinRequest.query.get(join_request_parts[-1]) - existing_membership = CommunityMember.query.filter_by(user_id=join_request.user_id, - community_id=join_request.community_id).first() - if not existing_membership: - member = CommunityMember(user_id=join_request.user_id, community_id=join_request.community_id) - db.session.add(member) - community.subscriptions_count += 1 - db.session.commit() - cache.delete_memoized(community_membership, User.query.get(join_request.user_id), Community.query.get(join_request.community_id)) - activity_log.result = 'success' - elif request_json['object']['type'] == 'Follow': - community_ap_id = request_json['actor'] - user_ap_id = request_json['object']['actor'] - user = find_actor_or_create(user_ap_id) - community = find_actor_or_create(community_ap_id, community_only=True) - if user and community: - join_request = CommunityJoinRequest.query.filter_by(user_id=user.id, community_id=community.id).first() - if join_request: - existing_membership = CommunityMember.query.filter_by(user_id=user.id, community_id=community.id).first() - if not existing_membership: - member = CommunityMember(user_id=user.id, community_id=community.id) - db.session.add(member) - community.subscriptions_count += 1 - db.session.commit() - activity_log.result = 'success' - cache.delete_memoized(community_membership, user, community) - - elif request_json['type'] == 'Undo': - if request_json['object']['type'] == 'Follow': # Unsubscribe from a community - community_ap_id = request_json['object']['object'] - user_ap_id = request_json['object']['actor'] - user = find_actor_or_create(user_ap_id) - community = find_actor_or_create(community_ap_id, community_only=True) - if user and community: - user.last_seen = utcnow() - member = CommunityMember.query.filter_by(user_id=user.id, community_id=community.id).first() - join_request = CommunityJoinRequest.query.filter_by(user_id=user.id, community_id=community.id).first() - if member: - db.session.delete(member) - community.subscriptions_count -= 1 - if join_request: - db.session.delete(join_request) - db.session.commit() - cache.delete_memoized(community_membership, user, community) - activity_log.result = 'success' - elif request_json['object']['type'] == 'Like': # Undoing an upvote or downvote - activity_log.activity_type = request_json['object']['type'] - user_ap_id = request_json['actor'] - user = find_actor_or_create(user_ap_id) - post = None - comment = None - target_ap_id = request_json['object']['object'] - post_or_comment = undo_vote(activity_log, comment, post, target_ap_id, user) - if post_or_comment: - announce_activity_to_followers(post_or_comment.community, user, request_json) - activity_log.result = 'success' - elif request_json['object']['type'] == 'Dislike': # Undoing a downvote - probably unused - activity_log.activity_type = request_json['object']['type'] - user_ap_id = request_json['actor'] - user = find_actor_or_create(user_ap_id) - post = None - comment = None - target_ap_id = request_json['object']['object'] - post_or_comment = undo_downvote(activity_log, comment, post, target_ap_id, user) - if post_or_comment: - announce_activity_to_followers(post_or_comment.community, user, request_json) - activity_log.result = 'success' - elif request_json['object']['type'] == 'Block': # Undoing a ban - activity_log.activity_type = 'Undo User Ban' - deletor_ap_id = request_json['object']['actor'] - user_ap_id = request_json['object']['object'] - target = request_json['object']['target'] - if user_ap_id.startswith('https://' + current_app.config['SERVER_NAME']): - unban_local_user(deletor_ap_id, user_ap_id, target) - activity_log.result = 'success' - elif request_json['object']['type'] == 'Delete': # undoing a delete - activity_log.activity_type = 'Restore' - post = Post.query.filter_by(ap_id=request_json['object']['object']).first() - if post: - deletor = find_actor_or_create(request_json['object']['actor'], create_if_not_found=False) - if deletor: - if post.author.id == deletor.id or post.community.is_moderator(deletor) or post.community.is_instance_admin(deletor): - post.deleted = False - post.deleted_by = None - post.author.post_count += 1 - post.community.post_count += 1 - announce_activity_to_followers(post.community, post.author, request_json) - db.session.commit() - activity_log.result = 'success' - else: - activity_log.exception_message = 'Restore attempt denied' - else: - activity_log.exception_message = 'Restorer did not already exist' - else: - reply = PostReply.query.filter_by(ap_id=request_json['object']['object']).first() - if reply: - deletor = find_actor_or_create(request_json['object']['actor'], create_if_not_found=False) - if deletor: - if reply.author.id == deletor.id or reply.community.is_moderator(deletor) or reply.community.is_instance_admin(deletor): - reply.deleted = False - reply.deleted_by = None - if not reply.author.bot: - reply.post.reply_count += 1 - reply.author.post_reply_count += 1 - announce_activity_to_followers(reply.community, reply.author, request_json) - db.session.commit() - activity_log.result = 'success' - else: - activity_log.exception_message = 'Restore attempt denied' - else: - activity_log.exception_message = 'Restorer did not already exist' - else: - activity_log.exception_message = 'Object not found, or object was not a post or a reply' - elif request_json['type'] == 'Delete': - if isinstance(request_json['object'], str): - ap_id = request_json['object'] # lemmy - else: - ap_id = request_json['object']['id'] # kbin - post = Post.query.filter_by(ap_id=ap_id).first() - # Delete post - if post: - deletor = find_actor_or_create(request_json['actor'], create_if_not_found=False) - if deletor: - if post.author.id == deletor.id or post.community.is_moderator(deletor) or post.community.is_instance_admin(deletor): - post.deleted = True - post.delted_by = deletor.id - post.author.post_count -= 1 - post.community.post_count -= 1 - if post.url and post.cross_posts is not None: - old_cross_posts = Post.query.filter(Post.id.in_(post.cross_posts)).all() - post.cross_posts.clear() - for ocp in old_cross_posts: - if ocp.cross_posts is not None: - ocp.cross_posts.remove(post.id) - announce_activity_to_followers(post.community, post.author, request_json) - db.session.commit() - activity_log.result = 'success' - else: - activity_log.exception_message = 'Delete attempt denied' - else: - activity_log.exception_message = 'Deletor did not already exist' - else: - # Delete PostReply - reply = PostReply.query.filter_by(ap_id=ap_id).first() - if reply: - deletor = find_actor_or_create(request_json['actor'], create_if_not_found=False) - if deletor: - if reply.author.id == deletor.id or reply.community.is_moderator(deletor) or reply.community.is_instance_admin(deletor): - reply.deleted = True - reply.deleted_by = deletor.id - if not reply.author.bot: - reply.post.reply_count -= 1 - reply.author.post_reply_count -= 1 - announce_activity_to_followers(reply.community, reply.author, request_json) - db.session.commit() - activity_log.result = 'success' - else: - activity_log.exception_message = 'Delete attempt denied' - else: - activity_log.exception_message = 'Deletor did not already exist' - else: - # Delete User - user = find_actor_or_create(ap_id, create_if_not_found=False) - if user: - user.deleted = True - user.delete_dependencies() - db.session.commit() - activity_log.result = 'success' - else: - activity_log.exception_message = 'Delete: cannot find ' + ap_id - - elif request_json['type'] == 'Like' or request_json['type'] == 'EmojiReact': # Upvote - activity_log.activity_type = request_json['type'] - user_ap_id = request_json['actor'] - user = find_actor_or_create(user_ap_id) - liked = find_liked_object(request_json['object']) - if user is None: - activity_log.exception_message = 'Blocked or unfound user' - elif liked is None: - activity_log.exception_message = 'Unfound object ' + request_json['object'] - elif user.is_local(): - activity_log.exception_message = 'Activity about local content which is already present' - activity_log.result = 'ignored' - elif can_upvote(user, liked.community): - # insert into voted table - if liked is None: - activity_log.exception_message = 'Liked object not found' - elif liked is not None and isinstance(liked, (Post, PostReply)): - liked.vote(user, 'upvote') - activity_log.result = 'success' - else: - activity_log.exception_message = 'Could not detect type of like' - if activity_log.result == 'success': - announce_activity_to_followers(liked.community, user, request_json) - else: - activity_log.exception_message = 'Cannot upvote this' - activity_log.result = 'ignored' - elif request_json['type'] == 'Dislike': # Downvote - if get_setting('allow_dislike', True) is False: - activity_log.exception_message = 'Dislike ignored because of allow_dislike setting' - else: - activity_log.activity_type = request_json['type'] - user_ap_id = request_json['actor'] - user = find_actor_or_create(user_ap_id) - target_ap_id = request_json['object'] - disliked = find_liked_object(target_ap_id) - if user is None: - activity_log.exception_message = 'Blocked or unfound user' - elif disliked is None: - activity_log.exception_message = 'Unfound object' + target_ap_id - elif user.is_local(): - activity_log.exception_message = 'Activity about local content which is already present' - activity_log.result = 'ignored' - elif can_downvote(user, disliked.community, site): - # insert into voted table - if disliked is None: - activity_log.exception_message = 'Liked object not found' - elif isinstance(disliked, (Post, PostReply)): - disliked.vote(user, 'downvote') - activity_log.result = 'success' - else: - activity_log.exception_message = 'Could not detect type of like' - if activity_log.result == 'success': - announce_activity_to_followers(disliked.community, user, request_json) - else: - activity_log.exception_message = 'Cannot downvote this' - activity_log.result = 'ignored' - elif request_json['type'] == 'Flag': # Reported content - activity_log.activity_type = 'Report' - user_ap_id = request_json['actor'] - user = find_actor_or_create(user_ap_id) - target_ap_id = request_json['object'] - reported = find_reported_object(target_ap_id) - if user and reported: - process_report(user, reported, request_json, activity_log) - announce_activity_to_followers(reported.community, user, request_json) - activity_log.result = 'success' - else: - activity_log.exception_message = 'Report ignored due to missing user or content' - elif request_json['type'] == 'Block': - activity_log.activity_type = 'User Ban' - deletor_ap_id = request_json['actor'] - user_ap_id = request_json['object'] - target = request_json['target'] - remove_data = request_json['removeData'] - if remove_data == True: - remove_data_from_banned_user(deletor_ap_id, user_ap_id, target) - if user_ap_id.startswith('https://' + current_app.config['SERVER_NAME']): - ban_local_user(deletor_ap_id, user_ap_id, target, request_json) - activity_log.result = 'success' - - # Flush the caches of any major object that was created. To be sure. - if 'user' in vars() and user is not None: - if user.instance_id and user.instance_id != 1: - user.instance.last_seen = utcnow() - # user.instance.ip_address = ip_address - user.instance.dormant = False - user.instance.gone_forever = False - user.instance.failures = 0 - else: - activity_log.exception_message = 'Instance blocked' - - if activity_log.exception_message is not None and activity_log.result == 'processing': - activity_log.result = 'failure' - # Don't log successful json - save space - if site.log_activitypub_json and activity_log.result == 'success' and not current_app.debug: - activity_log.activity_json = '' - db.session.commit() + log_incoming_ap(request_json['id'], APLOG_MONITOR, APLOG_PROCESSING, request_json if store_ap_json else None, 'Unmatched activity') @celery.task From cc282fe5f7cf18df5e5ed6318dc1bdc7bf76b630 Mon Sep 17 00:00:00 2001 From: freamon Date: Sun, 24 Nov 2024 21:49:08 +0000 Subject: [PATCH 34/36] apf part 34: Redirect user inbox to shared inbox --- app/activitypub/routes.py | 104 ++++---------------------------------- 1 file changed, 11 insertions(+), 93 deletions(-) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index 5b5e0c5f..028490b8 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -491,11 +491,21 @@ def shared_inbox(): return '' -@bp.route('/site_inbox', methods=['GET', 'POST']) +@bp.route('/site_inbox', methods=['POST']) def site_inbox(): return shared_inbox() +@bp.route('/u//inbox', methods=['POST']) +def user_inbox(actor): + return shared_inbox() + + +@bp.route('/c//inbox', methods=['POST']) +def community_inbox(actor): + return shared_inbox() + + @celery.task def process_inbox_request(request_json, store_ap_json): with current_app.app_context(): @@ -1225,93 +1235,6 @@ def community_moderators_route(actor): return jsonify(community_data) -@bp.route('/u//inbox', methods=['POST']) -def user_inbox(actor): - site = Site.query.get(1) - activity_log = ActivityPubLog(direction='in', result='failure') - activity_log.result = 'processing' - db.session.add(activity_log) - db.session.commit() - - try: - request_json = request.get_json(force=True) - except werkzeug.exceptions.BadRequest as e: - activity_log.exception_message = 'Unable to parse json body: ' + e.description - activity_log.result = 'failure' - db.session.commit() - return '', 400 - - if 'id' in request_json: - activity_log.activity_id = request_json['id'] - if site.log_activitypub_json: - activity_log.activity_json = json.dumps(request_json) - - actor = find_actor_or_create(request_json['actor'], signed_get=True) if 'actor' in request_json else None - if actor is not None: - if (('type' in request_json and request_json['type'] == 'Like') or - ('type' in request_json and request_json['type'] == 'Undo' and - 'object' in request_json and request_json['object']['type'] == 'Like')): - return shared_inbox() - if 'type' in request_json and request_json['type'] == 'Accept': - return shared_inbox() - try: - HttpSignature.verify_request(request, actor.public_key, skip_date=True) - if 'type' in request_json: - if request_json['type'] == 'Follow': - if current_app.debug: - process_user_follow_request(request_json, activity_log.id, actor.id) - else: - process_user_follow_request.delay(request_json, activity_log.id, actor.id) - elif request_json['type'] == 'Undo' and 'object' in request_json and request_json['object']['type'] == 'Follow': - local_user_ap_id = request_json['object']['object'] - local_user = find_actor_or_create(local_user_ap_id, create_if_not_found=False) - remote_user = User.query.get(actor.id) - if local_user: - db.session.query(UserFollower).filter_by(local_user_id=local_user.id, remote_user_id=remote_user.id, is_accepted=True).delete() - activity_log.result = 'success' - else: - activity_log.exception_message = 'Could not find local user' - activity_log.result = 'failure' - db.session.commit() - elif ('type' in request_json and request_json['type'] == 'Create' and - 'object' in request_json and request_json['object']['type'] == 'Note' and - 'name' in request_json['object']): # poll votes - in_reply_to = request_json['object']['inReplyTo'] if 'inReplyTo' in request_json['object'] else None - if in_reply_to: - post_being_replied_to = Post.query.filter_by(ap_id=request_json['object']['inReplyTo']).first() - if post_being_replied_to: - community_ap_id = post_being_replied_to.community.ap_profile_id - community = find_actor_or_create(community_ap_id, community_only=True, create_if_not_found=False) - user_ap_id = request_json['object']['attributedTo'] - user = find_actor_or_create(user_ap_id, create_if_not_found=False) - if can_create_post_reply(user, community): - poll_data = Poll.query.get(post_being_replied_to.id) - choice = PollChoice.query.filter_by(post_id=post_being_replied_to.id, choice_text=request_json['object']['name']).first() - if poll_data and choice: - poll_data.vote_for_choice(choice.id, user.id) - activity_log.activity_type = 'Poll Vote' - activity_log.result = 'success' - db.session.commit() - if post_being_replied_to.author.is_local(): - inform_followers_of_post_update(post_being_replied_to.id, user.instance_id) - - except VerificationError: - activity_log.result = 'failure' - activity_log.exception_message = 'Could not verify signature' - db.session.commit() - return '', 400 - else: - actor_name = request_json['actor'] if 'actor' in request_json else '' - activity_log.exception_message = f'Actor could not be found 2: {actor_name}' - - if activity_log.exception_message is not None: - activity_log.result = 'failure' - db.session.commit() - resp = jsonify('ok') - resp.content_type = 'application/activity+json' - return resp - - @celery.task def process_user_follow_request(request_json, activitypublog_id, remote_user_id): activity_log = ActivityPubLog.query.get(activitypublog_id) @@ -1354,11 +1277,6 @@ def process_user_follow_request(request_json, activitypublog_id, remote_user_id) db.session.commit() -@bp.route('/c//inbox', methods=['GET', 'POST']) -def community_inbox(actor): - return shared_inbox() - - @bp.route('/c//followers', methods=['GET']) def community_followers(actor): actor = actor.strip() From ff90e503321fb51b1a50cbc464eb7cab70043667 Mon Sep 17 00:00:00 2001 From: freamon Date: Sun, 24 Nov 2024 22:01:11 +0000 Subject: [PATCH 35/36] apf part 35: Replay inbox requests --- app/activitypub/routes.py | 58 +++++++++++++++++++++++++++++++++++++++ app/admin/routes.py | 10 +++---- 2 files changed, 62 insertions(+), 6 deletions(-) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index 028490b8..4c5cccdb 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -506,6 +506,64 @@ def community_inbox(actor): return shared_inbox() +def replay_inbox_request(request_json): + if not 'id' in request_json or not 'type' in request_json or not 'actor' in request_json or not 'object' in request_json: + log_incoming_ap('', APLOG_NOTYPE, APLOG_FAILURE, request_json, 'REPLAY: Missing minimum expected fields in JSON') + return + + id = request_json['id'] + if request_json['type'] == 'Announce' and isinstance(request_json['object'], dict): + object = request_json['object'] + if not 'id' in object or not 'type' in object or not 'actor' in object or not 'object' in object: + if object['type'] == 'Page': + log_incoming_ap(request_json['id'], APLOG_ANNOUNCE, APLOG_IGNORED, request_json, 'REPLAY: Intended for Mastodon') + elif object['type'] == 'Note': # Lemmy has Announced Note out from Mastodon, but will also announce Create/Note + log_incoming_ap(request_json['id'], APLOG_ANNOUNCE, APLOG_IGNORED, request_json, 'REPLAY: Intended for Mastodon') + else: + log_incoming_ap(request_json['id'], APLOG_ANNOUNCE, APLOG_FAILURE, request_json, 'REPLAY: Missing minimum expected fields in JSON Announce object') + return + + actor = User.query.filter_by(ap_profile_id=object['actor'].lower()).first() + if actor and actor.is_local(): + log_incoming_ap(object['id'], APLOG_DUPLICATE, APLOG_IGNORED, request_json, 'REPLAY: Activity about local content which is already present') + + # Ignore unutilised PeerTube activity + if request_json['actor'].endswith('accounts/peertube'): + log_incoming_ap(request_json['id'], APLOG_PT_VIEW, APLOG_IGNORED, request_json, 'REPLAY: PeerTube View or CacheFile activity') + return + + # Ignore delete requests from uses that do not already exist here + if request_json['type'] == 'Delete': + if (request_json['id'].endswith('#delete') or # Mastodon / PieFed + ('object' in request_json and isinstance(request_json['object'], str) and request_json['actor'] == request_json['object'])): # Lemmy + actor = User.query.filter_by(ap_profile_id=request_json['actor'].lower()).first() + if not actor: + log_incoming_ap(request_json['id'], APLOG_DELETE, APLOG_IGNORED, request_json, 'REPLAY: Does not exist here') + return + + actor = find_actor_or_create(request_json['actor']) + if not actor: + actor_name = request_json['actor'] + log_incoming_ap(request_json['id'], APLOG_NOTYPE, APLOG_FAILURE, request_json, f'REPLAY: Actor could not be found 1: {actor_name}') + return + + if actor.is_local(): # should be impossible (can be Announced back, but not sent back without access to privkey) + log_incoming_ap(request_json['id'], APLOG_NOTYPE, APLOG_FAILURE, request_json, 'REPLAY: ActivityPub activity from a local actor') + return + + # When a user is deleted, the only way to be fairly sure they get deleted everywhere is to tell the whole fediverse. + # Earlier check means this is only for users that already exist, repeating it here means that http signature will have been verified + if request_json['type'] == 'Delete': + if (request_json['id'].endswith('#delete') or # Mastodon / PieFed + ('object' in request_json and isinstance(request_json['object'], str) and request_json['actor'] == request_json['object'])): # Lemmy + process_delete_request(request_json, True) + return + + process_inbox_request(request_json, True) + + return + + @celery.task def process_inbox_request(request_json, store_ap_json): with current_app.app_context(): diff --git a/app/admin/routes.py b/app/admin/routes.py index 36adb346..7fa2d189 100644 --- a/app/admin/routes.py +++ b/app/admin/routes.py @@ -12,7 +12,7 @@ from sqlalchemy import text, desc, or_ from PIL import Image from app import db, celery, cache -from app.activitypub.routes import process_inbox_request, process_delete_request +from app.activitypub.routes import process_inbox_request, process_delete_request, replay_inbox_request from app.activitypub.signature import post_request, default_context from app.activitypub.util import instance_allowed, instance_blocked, extract_domain_and_actor from app.admin.forms import FederationForm, SiteMiscForm, SiteProfileForm, EditCommunityForm, EditUserForm, \ @@ -611,10 +611,8 @@ def activity_json(activity_id): def activity_replay(activity_id): activity = ActivityPubLog.query.get_or_404(activity_id) request_json = json.loads(activity.activity_json) - if 'type' in request_json and request_json['type'] == 'Delete' and request_json['id'].endswith('#delete'): - process_delete_request(request_json, activity.id, None) - else: - process_inbox_request(request_json, activity.id, None) + replay_inbox_request(request_json) + return 'Ok' @@ -1379,4 +1377,4 @@ def admin_instance_edit(instance_id): joined_communities=joined_communities(current_user.get_id()), menu_topics=menu_topics(), site=g.site - ) \ No newline at end of file + ) From 26eb967a1c71003973818fae1f879f0fdfc7c34b Mon Sep 17 00:00:00 2001 From: freamon Date: Mon, 25 Nov 2024 12:46:34 +0000 Subject: [PATCH 36/36] apf part 36: Use signed get if remote site responds 401 --- app/activitypub/util.py | 45 ++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/app/activitypub/util.py b/app/activitypub/util.py index d36f0b70..0287d404 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -258,7 +258,7 @@ def instance_allowed(host: str) -> bool: return instance is not None -def find_actor_or_create(actor: str, create_if_not_found=True, community_only=False, signed_get=False) -> Union[User, Community, None]: +def find_actor_or_create(actor: str, create_if_not_found=True, community_only=False) -> Union[User, Community, None]: if isinstance(actor, dict): # Discourse does this actor = actor['id'] actor_url = actor.strip() @@ -316,34 +316,37 @@ def find_actor_or_create(actor: str, create_if_not_found=True, community_only=Fa else: # User does not exist in the DB, it's going to need to be created from it's remote home instance if create_if_not_found: if actor.startswith('https://'): - if not signed_get: + try: + actor_data = get_request(actor_url, headers={'Accept': 'application/activity+json'}) + except httpx.HTTPError: + time.sleep(randint(3, 10)) try: actor_data = get_request(actor_url, headers={'Accept': 'application/activity+json'}) - except httpx.HTTPError: - time.sleep(randint(3, 10)) - try: - actor_data = get_request(actor_url, headers={'Accept': 'application/activity+json'}) - except httpx.HTTPError as e: - raise e - return None - if actor_data.status_code == 200: - try: - actor_json = actor_data.json() - except Exception as e: - actor_data.close() - return None + except httpx.HTTPError as e: + raise e + return None + if actor_data.status_code == 200: + try: + actor_json = actor_data.json() + except Exception as e: actor_data.close() - actor_model = actor_json_to_model(actor_json, address, server) - if community_only and not isinstance(actor_model, Community): - return None - return actor_model - else: + return None + actor_data.close() + actor_model = actor_json_to_model(actor_json, address, server) + if community_only and not isinstance(actor_model, Community): + return None + return actor_model + elif actor_data.status_code == 401: try: site = Site.query.get(1) actor_data = signed_get_request(actor_url, site.private_key, f"https://{current_app.config['SERVER_NAME']}/actor#main-key") if actor_data.status_code == 200: - actor_json = actor_data.json() + try: + actor_json = actor_data.json() + except Exception as e: + actor_data.close() + return None actor_data.close() actor_model = actor_json_to_model(actor_json, address, server) if community_only and not isinstance(actor_model, Community):