From e45c8d9db3a590749b1bcbaa90c463a465942453 Mon Sep 17 00:00:00 2001 From: freamon Date: Sun, 19 Jan 2025 15:04:32 +0000 Subject: [PATCH 01/16] Also use search.retrieve_remote_post() for requests from community side bar --- app/community/routes.py | 21 +-------------------- app/templates/_side_pane.html | 2 +- 2 files changed, 2 insertions(+), 21 deletions(-) diff --git a/app/community/routes.py b/app/community/routes.py index 319a469a..f0b12b21 100644 --- a/app/community/routes.py +++ b/app/community/routes.py @@ -21,8 +21,7 @@ from app.chat.util import send_message from app.community.forms import SearchRemoteCommunity, CreateDiscussionForm, CreateImageForm, CreateLinkForm, \ ReportCommunityForm, \ DeleteCommunityForm, AddCommunityForm, EditCommunityForm, AddModeratorForm, BanUserCommunityForm, \ - EscalateReportForm, ResolveReportForm, CreateVideoForm, CreatePollForm, RetrieveRemotePost, \ - EditCommunityWikiPageForm + EscalateReportForm, ResolveReportForm, CreateVideoForm, CreatePollForm, EditCommunityWikiPageForm from app.community.util import search_for_community, actor_to_community, \ save_icon_file, save_banner_file, send_to_remote_instance, \ delete_post_from_community, delete_post_reply_from_community, community_in_list, find_local_users, tags_from_string, \ @@ -152,24 +151,6 @@ def add_remote(): site=g.site) -@bp.route('/retrieve_remote_post/', methods=['GET', 'POST']) -@login_required -def retrieve_remote_post(community_id: int): - if current_user.banned: - return show_ban_message() - form = RetrieveRemotePost() - new_post = None - community = Community.query.get_or_404(community_id) - if form.validate_on_submit(): - address = form.address.data.strip() - new_post = resolve_remote_post(address, community_id) - if new_post is None: - flash(_('Post not found.'), 'warning') - - return render_template('community/retrieve_remote_post.html', - title=_('Retrieve Remote Post'), form=form, new_post=new_post, community=community) - - # @bp.route('/c/', methods=['GET']) - defined in activitypub/routes.py, which calls this function for user requests. A bit weird. def show_community(community: Community): diff --git a/app/templates/_side_pane.html b/app/templates/_side_pane.html index d067159a..face4b8a 100644 --- a/app/templates/_side_pane.html +++ b/app/templates/_side_pane.html @@ -52,7 +52,7 @@ {% if rss_feed and not community.is_local() -%} {% endif -%} {% if community.local_only -%} From 5bf0ddc32ad577ee24c85dd4dfa05184a04de784 Mon Sep 17 00:00:00 2001 From: freamon Date: Sun, 19 Jan 2025 16:29:16 +0000 Subject: [PATCH 02/16] Only call resolve_remote_post() from activitypub, modify it to also accept Announce/Update, and remove reliance on 'audience' field (#427) --- app/activitypub/routes.py | 2 +- app/activitypub/util.py | 168 +++++++++++++++++++------------------- 2 files changed, 85 insertions(+), 85 deletions(-) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index 24afb5ce..8d2845e6 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -614,7 +614,7 @@ def process_inbox_request(request_json, store_ap_json): if request_json['object'].startswith('https://' + current_app.config['SERVER_NAME']): log_incoming_ap(id, APLOG_DUPLICATE, APLOG_IGNORED, saved_json, 'Activity about local content which is already present') return - post = resolve_remote_post(request_json['object'], community.id, announce_actor=community.ap_profile_id, store_ap_json=store_ap_json) + post = resolve_remote_post(request_json['object'], community, id, store_ap_json) if post: log_incoming_ap(id, APLOG_ANNOUNCE, APLOG_SUCCESS, request_json) else: diff --git a/app/activitypub/util.py b/app/activitypub/util.py index 5ba58ff0..ebb3dda3 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -2295,93 +2295,86 @@ def can_delete(user_ap_id, post): return can_edit(user_ap_id, post) -def resolve_remote_post(uri: str, community_id: int, announce_actor=None, store_ap_json=False) -> Union[Post, PostReply, None]: - post = Post.query.filter_by(ap_id=uri).first() - if post: - return post - - community = Community.query.get(community_id) - site = Site.query.get(1) - +# called from incoming activitypub, when the object in an Announce is just a URL +# despite the name, it works for both posts and replies +def resolve_remote_post(uri: str, community, announce_id, store_ap_json) -> Union[Post, PostReply, None]: parsed_url = urlparse(uri) uri_domain = parsed_url.netloc - if announce_actor: - parsed_url = urlparse(announce_actor) - announce_actor_domain = parsed_url.netloc - if announce_actor_domain != 'a.gup.pe' and announce_actor_domain != uri_domain: - return None + announce_actor = community.ap_profile_id + parsed_url = urlparse(announce_actor) + announce_actor_domain = parsed_url.netloc + if announce_actor_domain != 'a.gup.pe' and announce_actor_domain != uri_domain: + return None actor_domain = None actor = None - post_request = get_request(uri, headers={'Accept': 'application/activity+json'}) - if post_request.status_code == 200: - post_data = post_request.json() - post_request.close() - # check again that it doesn't already exist (can happen with different but equivalent URLs) - post = Post.query.filter_by(ap_id=post_data['id']).first() - if post: - return post - if 'attributedTo' in post_data: - if isinstance(post_data['attributedTo'], str): - actor = post_data['attributedTo'] - parsed_url = urlparse(post_data['attributedTo']) - actor_domain = parsed_url.netloc - elif isinstance(post_data['attributedTo'], list): - for a in post_data['attributedTo']: - if a['type'] == 'Person': - actor = a['id'] - parsed_url = urlparse(a['id']) - actor_domain = parsed_url.netloc - break - if uri_domain != actor_domain: + + try: + object_request = get_request(uri, headers={'Accept': 'application/activity+json'}) + except httpx.HTTPError: + time.sleep(3) + try: + object_request = get_request(uri, headers={'Accept': 'application/activity+json'}) + except httpx.HTTPError: return None + if object_request.status_code == 200: + try: + post_data = object_request.json() + except: + object_request.close() + return None + object_request.close() + elif object_request.status_code == 401: + try: + site = Site.query.get(1) + object_request = signed_get_request(uri, site.private_key, f"https://{current_app.config['SERVER_NAME']}/actor#main-key") + except httpx.HTTPError: + time.sleep(3) + try: + object_request = signed_get_request(uri, site.private_key, f"https://{current_app.config['SERVER_NAME']}/actor#main-key") + except httpx.HTTPError: + return None + try: + post_data = object_request.json() + except: + object_request.close() + return None + object_request.close() + else: + return None - if not announce_actor: - # make sure that the post actually belongs in the community a user says it does - remote_community = None - if post_data['type'] == 'Page': # lemmy - remote_community = post_data['audience'] if 'audience' in post_data else None - if remote_community and remote_community.lower() != community.ap_profile_id: - return None - elif post_data['type'] == 'Video': # peertube - if 'attributedTo' in post_data and isinstance(post_data['attributedTo'], list): - for a in post_data['attributedTo']: - if a['type'] == 'Group': - remote_community = a['id'] - break - if remote_community and remote_community.lower() != community.ap_profile_id: - return None - else: # mastodon, etc - if 'inReplyTo' not in post_data or post_data['inReplyTo'] != None: - return None - community_found = False - if not community_found and 'to' in post_data and isinstance(post_data['to'], str): - remote_community = post_data['to'] - if remote_community.lower() == community.ap_profile_id: - community_found = True - if not community_found and 'cc' in post_data and isinstance(post_data['cc'], str): - remote_community = post_data['cc'] - if remote_community.lower() == community.ap_profile_id: - community_found = True - if not community_found and 'to' in post_data and isinstance(post_data['to'], list): - for t in post_data['to']: - if t.lower() == community.ap_profile_id: - community_found = True - break - if not community_found and 'cc' in post_data and isinstance(post_data['cc'], list): - for c in post_data['cc']: - if c.lower() == community.ap_profile_id: - community_found = True - break - if not community_found: - return None + # find the author. Make sure their domain matches the site hosting it to mitigate impersonation attempts + if 'attributedTo' in post_data: + attributed_to = post_data['attributedTo'] + if isinstance(attributed_to, str): + actor = attributed_to + parsed_url = urlparse(actor) + actor_domain = parsed_url.netloc + elif isinstance(attributed_to, list): + for a in attributed_to: + if isinstance(a, dict) and a.get('type') == 'Person': + actor = a.get('id') + if isinstance(actor, str): # Ensure `actor` is a valid string + parsed_url = urlparse(actor) + actor_domain = parsed_url.netloc + break + elif isinstance(a, str): + actor = a + parsed_url = urlparse(actor) + actor_domain = parsed_url.netloc + break + if uri_domain != actor_domain: + return None - user = find_actor_or_create(actor) - if user and community and post_data: - request_json = { - 'id': f"https://{uri_domain}/activities/create/{gibberish(15)}", - 'object': post_data - } - if 'inReplyTo' in request_json['object'] and request_json['object']['inReplyTo']: + user = find_actor_or_create(actor) + if user and community and post_data: + activity = 'update' if 'updated' in post_data else 'create' + request_json = {'id': f"https://{uri_domain}/activities/{activity}/{gibberish(15)}", 'object': post_data} + if 'inReplyTo' in request_json['object'] and request_json['object']['inReplyTo']: + if activity == 'update': + post_reply = PostReply.get_by_ap_id(uri) + if post_reply: + update_post_reply_from_activity(post_reply, request_json) + else: post_reply = create_post_reply(store_ap_json, community, request_json['object']['inReplyTo'], request_json, user) if post_reply: if 'published' in post_data: @@ -2389,21 +2382,28 @@ def resolve_remote_post(uri: str, community_id: int, announce_actor=None, store_ post_reply.post.last_active = post_data['published'] post_reply.community.last_active = utcnow() db.session.commit() - return post_reply + if post_reply: + return post_reply + else: + if activity == 'update': + post = Post.get_by_ap_id(uri) + if post: + update_post_from_activity(post_reply, request_json) else: - post = create_post(store_ap_json, community, request_json, user) + post = create_post(store_ap_json, community, request_json, user, announce_id) if post: if 'published' in post_data: post.posted_at=post_data['published'] post.last_active=post_data['published'] post.community.last_active = utcnow() db.session.commit() - return post + if post: + return post return None -# called from UI, via 'search' option in navbar +# called from UI, via 'search' option in navbar, or 'Retrieve a post from the original server' in community sidebar def resolve_remote_post_from_search(uri: str) -> Union[Post, None]: post = Post.query.filter_by(ap_id=uri).first() if post: From 1654dcea0b7b6374039ce81548bc2b667646a15c Mon Sep 17 00:00:00 2001 From: freamon Date: Sun, 19 Jan 2025 19:31:31 +0000 Subject: [PATCH 03/16] Consolidate retrieve_mods_and_backfill() functions to support outbox processing for lemmy, peertube, and wordpress --- app/community/util.py | 270 ++++++++++++++++++++++-------------------- 1 file changed, 139 insertions(+), 131 deletions(-) diff --git a/app/community/util.py b/app/community/util.py index 11f54780..50e23023 100644 --- a/app/community/util.py +++ b/app/community/util.py @@ -10,9 +10,9 @@ from flask_login import current_user from pillow_heif import register_heif_opener from app import db, cache, celery -from app.activitypub.signature import post_request, default_context +from app.activitypub.signature import post_request, default_context, signed_get_request from app.activitypub.util import find_actor_or_create, actor_json_to_model, post_json_to_model, ensure_domains_match, \ - find_hashtag_or_create + find_hashtag_or_create, create_post from app.constants import POST_TYPE_ARTICLE, POST_TYPE_LINK, POST_TYPE_IMAGE, POST_TYPE_VIDEO, NOTIF_POST, \ POST_TYPE_POLL from app.models import Community, File, BannedInstances, PostReply, Post, utcnow, CommunityMember, Site, \ @@ -71,159 +71,167 @@ def search_for_community(address: str): if community_json['type'] == 'Group': community = actor_json_to_model(community_json, name, server) if community: - if community.ap_profile_id == f"https://{server}/video-channels/{name}": - if current_app.debug: - retrieve_peertube_mods_and_backfill(community.id, community_json['attributedTo']) - else: - retrieve_peertube_mods_and_backfill.delay(community.id, community_json['attributedTo']) - return community if current_app.debug: - retrieve_mods_and_backfill(community.id) + retrieve_mods_and_backfill(community.id, server, name, community_json) else: - retrieve_mods_and_backfill.delay(community.id) + retrieve_mods_and_backfill.delay(community.id, server, name, community_json) return community return None -@celery.task -def retrieve_peertube_mods_and_backfill(community_id: int, mods: list): - community = Community.query.get(community_id) - site = Site.query.get(1) - for m in mods: - user = find_actor_or_create(m['id']) - if user: - existing_membership = CommunityMember.query.filter_by(community_id=community.id, user_id=user.id).first() - if existing_membership: - existing_membership.is_moderator = True - else: - new_membership = CommunityMember(community_id=community.id, user_id=user.id, is_moderator=True) - db.session.add(new_membership) - community.restricted_to_mods = True - db.session.commit() - - if community.ap_public_url: - outbox_request = get_request(community.ap_outbox_url, headers={'Accept': 'application/activity+json'}) - if outbox_request.status_code == 200: - outbox_data = outbox_request.json() - outbox_request.close() - if 'totalItems' in outbox_data and outbox_data['totalItems'] > 0: - page1_request = get_request(outbox_data['first'], headers={'Accept': 'application/activity+json'}) - if page1_request.status_code == 200: - page1_data = page1_request.json() - page1_request.close() - if 'type' in page1_data and page1_data['type'] == 'OrderedCollectionPage' and 'orderedItems' in page1_data: - # only 10 posts per page for PeerTube - for activity in page1_data['orderedItems']: - video_request = get_request(activity['object'], headers={'Accept': 'application/activity+json'}) - if video_request.status_code == 200: - video_data = video_request.json() - video_request.close() - activity_log = ActivityPubLog(direction='in', activity_id=video_data['id'], activity_type='Video', result='failure') - if site.log_activitypub_json: - activity_log.activity_json = json.dumps(video_data) - db.session.add(activity_log) - if not ensure_domains_match(video_data): - activity_log.exception_message = 'Domains do not match' - db.session.commit() - continue - if user and user.is_local(): - activity_log.exception_message = 'Activity about local content which is already present' - db.session.commit() - continue - if user: - post = post_json_to_model(activity_log, video_data, user, community) - post.ap_announce_id = activity['id'] - post.ranking = post.post_ranking(post.score, post.posted_at) - else: - activity_log.exception_message = 'Could not find or create actor' - db.session.commit() - if community.post_count > 0: - community.last_active = Post.query.filter(Post.community_id == community_id).order_by(desc(Post.posted_at)).first().posted_at - db.session.commit() +def remote_object_to_json(uri): + try: + object_request = get_request(uri, headers={'Accept': 'application/activity+json'}) + except httpx.HTTPError: + time.sleep(3) + try: + object_request = get_request(uri, headers={'Accept': 'application/activity+json'}) + except httpx.HTTPError: + return None + if object_request.status_code == 200: + try: + object = object_request.json() + return object + except: + object_request.close() + return None + object_request.close() + elif object_request.status_code == 401: + try: + site = Site.query.get(1) + object_request = signed_get_request(uri, site.private_key, f"https://{current_app.config['SERVER_NAME']}/actor#main-key") + except httpx.HTTPError: + time.sleep(3) + try: + object_request = signed_get_request(uri, site.private_key, f"https://{current_app.config['SERVER_NAME']}/actor#main-key") + except httpx.HTTPError: + return None + try: + object = object_request.json() + return object + except: + object_request.close() + return None + object_request.close() + else: + return None @celery.task -def retrieve_mods_and_backfill(community_id: int): +def retrieve_mods_and_backfill(community_id: int, server, name, community_json=None): with current_app.app_context(): community = Community.query.get(community_id) + if not community: + return site = Site.query.get(1) - if community.ap_moderators_url: - mods_request = get_request(community.ap_moderators_url, headers={'Accept': 'application/activity+json'}) - if mods_request.status_code == 200: - mods_data = mods_request.json() - mods_request.close() - if mods_data and mods_data['type'] == 'OrderedCollection' and 'orderedItems' in mods_data: - for actor in mods_data['orderedItems']: - sleep(0.5) - user = find_actor_or_create(actor) - if user: - existing_membership = CommunityMember.query.filter_by(community_id=community.id, user_id=user.id).first() + + is_peertube = is_guppe = is_wordpress = False + if community.ap_profile_id == f"https://{server}/video-channels/{name}": + is_peertube = True + elif community.ap_profile_id.startswith('https://a.gup.pe/u'): + is_guppe = True + + # get mods + if community_json and 'attributedTo' in community_json: + mods = community_json['attributedTo'] + if isinstance(mods, list): + for m in mods: + if 'type' in m and m['type'] == 'Person' and 'id' in m: + mod = find_actor_or_create(m['id']) + if mod: + existing_membership = CommunityMember.query.filter_by(community_id=community.id, user_id=mod.id).first() if existing_membership: existing_membership.is_moderator = True else: - new_membership = CommunityMember(community_id=community.id, user_id=user.id, is_moderator=True) + new_membership = CommunityMember(community_id=community.id, user_id=mod.id, is_moderator=True) db.session.add(new_membership) - db.session.commit() + elif community.ap_moderators_url: + mods_data = remote_object_to_json(community.ap_moderators_url) + if mods_data and mods_data['type'] == 'OrderedCollection' and 'orderedItems' in mods_data: + for actor in mods_data['orderedItems']: + sleep(0.5) + mod = find_actor_or_create(actor) + if mod: + existing_membership = CommunityMember.query.filter_by(community_id=community.id, user_id=mod.id).first() + if existing_membership: + existing_membership.is_moderator = True + else: + new_membership = CommunityMember(community_id=community.id, user_id=mod.id, is_moderator=True) + db.session.add(new_membership) + if is_peertube: + community.restricted_to_mods = True + db.session.commit() # only backfill nsfw if nsfw communities are allowed if (community.nsfw and not site.enable_nsfw) or (community.nsfl and not site.enable_nsfl): return - # download 50 old posts + # download 50 old posts from unpaginated outboxes or 10 posts from page 1 if outbox is paginated (with Celery, or just 2 without) if community.ap_outbox_url: - outbox_request = get_request(community.ap_outbox_url, headers={'Accept': 'application/activity+json'}) - if outbox_request.status_code == 200: - outbox_data = outbox_request.json() - outbox_request.close() - if 'type' in outbox_data and outbox_data['type'] == 'OrderedCollection' and 'orderedItems' in outbox_data: - activities_processed = 0 - for activity in outbox_data['orderedItems']: - activity_log = ActivityPubLog(direction='in', activity_id=activity['id'], activity_type='Announce', result='failure') - if site.log_activitypub_json: - activity_log.activity_json = json.dumps(activity) - db.session.add(activity_log) - if 'object' in activity and 'object' in activity['object']: - if not ensure_domains_match(activity['object']['object']): - activity_log.exception_message = 'Domains do not match' - db.session.commit() - continue - user = find_actor_or_create(activity['object']['actor']) - if user and user.is_local(): - activity_log.exception_message = 'Activity about local content which is already present' - db.session.commit() - continue - if user: - post = post_json_to_model(activity_log, activity['object']['object'], user, community) - if post: - post.ap_create_id = activity['object']['id'] - post.ap_announce_id = activity['id'] - post.ranking = post.post_ranking(post.score, post.posted_at) - if post.url: - post.calculate_cross_posts() - db.session.commit() - else: - activity_log.exception_message = 'Could not find or create actor' + outbox_data = remote_object_to_json(community.ap_outbox_url) + if not outbox_data or ('totalItems' in outbox_data and outbox_data['totalItems'] == 0): + return + if 'first' in outbox_data: + outbox_data = remote_object_to_json(outbox_data['first']) + if not outbox_data: + return + max = 10 + else: + max = 50 + if current_app.debug: + max = 2 + if 'type' in outbox_data and (outbox_data['type'] == 'OrderedCollection' or outbox_data['type'] == 'OrderedCollectionPage') and 'orderedItems' in outbox_data: + activities_processed = 0 + for announce in outbox_data['orderedItems']: + activity = None + if is_peertube or is_guppe: + activity = remote_object_to_json(announce['object']) + elif 'object' in announce and 'object' in announce['object']: + activity = announce['object']['object'] + elif 'type' in announce and announce['type'] == 'Create': + activity = announce['object'] + is_wordpress = True + if not activity: + return + if not ensure_domains_match(activity): + continue + if is_peertube: + user = mod + elif 'attributedTo' in activity and isinstance(activity['attributedTo'], str): + user = find_actor_or_create(activity['attributedTo']) + if not user: + continue + else: + continue + if user.is_local(): + continue + if is_peertube or is_guppe: + request_json = {'id': f"https://{server}/activities/create/{gibberish(15)}", 'object': activity} + elif is_wordpress: + request_json = announce + else: + request_json = announce['object'] + post = create_post(True, community, request_json, user, announce['id']) + if post: + if 'published' in activity: + post.posted_at = activity['published'] + post.last_active = activity['published'] db.session.commit() - - activities_processed += 1 - if activities_processed >= 50: - break - c = Community.query.get(community.id) - if c.post_count > 0: - c.last_active = Post.query.filter(Post.community_id == community_id).order_by(desc(Post.posted_at)).first().posted_at + activities_processed += 1 + if activities_processed >= max: + break + if community.post_count > 0: + community.last_active = Post.query.filter(Post.community_id == community.id).order_by(desc(Post.posted_at)).first().posted_at db.session.commit() - if community.ap_featured_url: - featured_request = get_request(community.ap_featured_url, headers={'Accept': 'application/activity+json'}) - if featured_request.status_code == 200: - featured_data = featured_request.json() - featured_request.close() - if featured_data['type'] == 'OrderedCollection' and 'orderedItems' in featured_data: - for item in featured_data['orderedItems']: - featured_id = item['id'] - p = Post.query.filter(Post.ap_id == featured_id).first() - if p: - p.sticky = True - db.session.commit() + if community.ap_featured_url: + featured_data = remote_object_to_json(community.ap_featured_url) + if featured_data and 'type' in featured_data and featured_data['type'] == 'OrderedCollection' and 'orderedItems' in featured_data: + for item in featured_data['orderedItems']: + featured_id = item['id'] + p = Post.query.filter(Post.ap_id == featured_id).first() + if p: + p.sticky = True + db.session.commit() def actor_to_community(actor) -> Community: From 5765b9a29cefcdb58c80147fa091fdecc2af0689 Mon Sep 17 00:00:00 2001 From: freamon Date: Sun, 19 Jan 2025 19:33:51 +0000 Subject: [PATCH 04/16] Remove obsolete function --- app/activitypub/util.py | 114 ---------------------------------------- app/community/util.py | 2 +- 2 files changed, 1 insertion(+), 115 deletions(-) diff --git a/app/activitypub/util.py b/app/activitypub/util.py index ebb3dda3..608f1946 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -885,120 +885,6 @@ def actor_json_to_model(activity_json, address, server): return community -def post_json_to_model(activity_log, post_json, user, community) -> Post: - try: - nsfl_in_title = '[NSFL]' in post_json['name'].upper() or '(NSFL)' in post_json['name'].upper() - post = Post(user_id=user.id, community_id=community.id, - title=html.unescape(post_json['name']), - comments_enabled=post_json['commentsEnabled'] if 'commentsEnabled' in post_json else True, - sticky=post_json['stickied'] if 'stickied' in post_json else False, - nsfw=post_json['sensitive'], - nsfl=post_json['nsfl'] if 'nsfl' in post_json else nsfl_in_title, - ap_id=post_json['id'], - type=constants.POST_TYPE_ARTICLE, - posted_at=post_json['published'], - last_active=post_json['published'], - instance_id=user.instance_id, - indexable = user.indexable - ) - if 'content' in post_json: - if post_json['mediaType'] == 'text/html': - post.body_html = allowlist_html(post_json['content']) - if 'source' in post_json and post_json['source']['mediaType'] == 'text/markdown': - post.body = post_json['source']['content'] - post.body_html = markdown_to_html(post.body) # prefer Markdown if provided, overwrite version obtained from HTML - else: - post.body = html_to_text(post.body_html) - elif post_json['mediaType'] == 'text/markdown': - post.body = post_json['content'] - post.body_html = markdown_to_html(post.body) - if 'attachment' in post_json and len(post_json['attachment']) > 0 and 'type' in post_json['attachment'][0]: - alt_text = None - if post_json['attachment'][0]['type'] == 'Link': - post.url = post_json['attachment'][0]['href'] # Lemmy < 0.19.4 - if post_json['attachment'][0]['type'] == 'Image': - post.url = post_json['attachment'][0]['url'] # PieFed, Lemmy >= 0.19.4 - if 'name' in post_json['attachment'][0]: - alt_text = post_json['attachment'][0]['name'] - if post.url: - if is_image_url(post.url): - post.type = POST_TYPE_IMAGE - image = File(source_url=post.url) - if alt_text: - image.alt_text = alt_text - db.session.add(image) - post.image = image - elif is_video_url(post.url): - post.type = POST_TYPE_VIDEO - else: - post.type = POST_TYPE_LINK - post.url = remove_tracking_from_link(post.url) - domain = domain_from_url(post.url) - - # notify about links to banned websites. - already_notified = set() # often admins and mods are the same people - avoid notifying them twice - if domain: - if domain.notify_mods: - for community_member in post.community.moderators(): - notify = Notification(title='Suspicious content', url=post.ap_id, user_id=community_member.user_id, author_id=user.id) - db.session.add(notify) - already_notified.add(community_member.user_id) - - if domain.notify_admins: - for admin in Site.admins(): - if admin.id not in already_notified: - notify = Notification(title='Suspicious content', url=post.ap_id, user_id=admin.id, author_id=user.id) - db.session.add(notify) - admin.unread_notifications += 1 - if domain.banned: - post = None - activity_log.exception_message = domain.name + ' is blocked by admin' - if not domain.banned: - domain.post_count += 1 - post.domain = domain - - if post is not None: - if post_json['type'] == 'Video': - post.type = POST_TYPE_VIDEO - post.url = post_json['id'] - if 'icon' in post_json and isinstance(post_json['icon'], list): - icon = File(source_url=post_json['icon'][-1]['url']) - db.session.add(icon) - post.image = icon - - if 'language' in post_json: - language = find_language_or_create(post_json['language']['identifier'], post_json['language']['name']) - if language: - post.language_id = language.id - - if 'tag' in post_json: - for json_tag in post_json['tag']: - if json_tag['type'] == 'Hashtag': - # Lemmy adds the community slug as a hashtag on every post in the community, which we want to ignore - if json_tag['name'][1:].lower() != community.name.lower(): - hashtag = find_hashtag_or_create(json_tag['name']) - if hashtag: - post.tags.append(hashtag) - - if 'image' in post_json and post.image is None: - image = File(source_url=post_json['image']['url']) - db.session.add(image) - post.image = image - db.session.add(post) - community.post_count += 1 - user.post_count += 1 - activity_log.result = 'success' - db.session.commit() - - if post.image_id: - make_image_sizes(post.image_id, 170, 512, 'posts') # the 512 sized image is for masonry view - - return post - except KeyError as e: - current_app.logger.error(f'KeyError in post_json_to_model: ' + str(post_json)) - return None - - # Save two different versions of a File, after downloading it from file.source_url. Set a width parameter to None to avoid generating one of that size def make_image_sizes(file_id, thumbnail_width=50, medium_width=120, directory='posts', toxic_community=False): if current_app.debug: diff --git a/app/community/util.py b/app/community/util.py index 50e23023..c2a3dba8 100644 --- a/app/community/util.py +++ b/app/community/util.py @@ -11,7 +11,7 @@ from pillow_heif import register_heif_opener from app import db, cache, celery from app.activitypub.signature import post_request, default_context, signed_get_request -from app.activitypub.util import find_actor_or_create, actor_json_to_model, post_json_to_model, ensure_domains_match, \ +from app.activitypub.util import find_actor_or_create, actor_json_to_model, ensure_domains_match, \ find_hashtag_or_create, create_post from app.constants import POST_TYPE_ARTICLE, POST_TYPE_LINK, POST_TYPE_IMAGE, POST_TYPE_VIDEO, NOTIF_POST, \ POST_TYPE_POLL From e8d91769318380713c03d45b2acd363480e7e397 Mon Sep 17 00:00:00 2001 From: freamon Date: Sun, 19 Jan 2025 21:28:42 +0000 Subject: [PATCH 05/16] API: post locks --- app/api/alpha/routes.py | 14 ++++- app/api/alpha/utils/__init__.py | 2 +- app/api/alpha/utils/post.py | 15 ++++- app/shared/post.py | 34 +++++++++++- app/shared/tasks/__init__.py | 5 +- app/shared/tasks/locks.py | 97 +++++++++++++++++++++++++++++++++ 6 files changed, 161 insertions(+), 6 deletions(-) create mode 100644 app/shared/tasks/locks.py diff --git a/app/api/alpha/routes.py b/app/api/alpha/routes.py index b6dcc8a2..ef03ebe8 100644 --- a/app/api/alpha/routes.py +++ b/app/api/alpha/routes.py @@ -1,7 +1,7 @@ from app.api.alpha import bp from app.api.alpha.utils import get_site, post_site_block, \ get_search, \ - get_post_list, get_post, post_post_like, put_post_save, put_post_subscribe, post_post, put_post, post_post_delete, post_post_report, \ + get_post_list, get_post, post_post_like, put_post_save, put_post_subscribe, post_post, put_post, post_post_delete, post_post_report, post_post_lock, \ get_reply_list, post_reply_like, put_reply_save, put_reply_subscribe, post_reply, put_reply, post_reply_delete, post_reply_report, \ get_community_list, get_community, post_community_follow, post_community_block, \ get_user, post_user_block @@ -207,6 +207,17 @@ def post_alpha_post_report(): return jsonify({"error": str(ex)}), 400 +@bp.route('/api/alpha/post/lock', methods=['POST']) +def post_alpha_post_lock(): + if not enable_api(): + return jsonify({'error': 'alpha api is not enabled'}) + try: + auth = request.headers.get('Authorization') + data = request.get_json(force=True) or {} + return jsonify(post_post_lock(auth, data)) + except Exception as ex: + return jsonify({"error": str(ex)}), 400 + # Reply @bp.route('/api/alpha/comment/list', methods=['GET']) def get_alpha_comment_list(): @@ -370,7 +381,6 @@ def alpha_community(): # Post - not yet implemented @bp.route('/api/alpha/post/remove', methods=['POST']) -@bp.route('/api/alpha/post/lock', methods=['POST']) @bp.route('/api/alpha/post/feature', methods=['POST']) @bp.route('/api/alpha/post/report', methods=['POST']) @bp.route('/api/alpha/post/report/resolve', methods=['PUT']) diff --git a/app/api/alpha/utils/__init__.py b/app/api/alpha/utils/__init__.py index 7f9d1d21..37d627e0 100644 --- a/app/api/alpha/utils/__init__.py +++ b/app/api/alpha/utils/__init__.py @@ -1,6 +1,6 @@ from app.api.alpha.utils.site import get_site, post_site_block from app.api.alpha.utils.misc import get_search -from app.api.alpha.utils.post import get_post_list, get_post, post_post_like, put_post_save, put_post_subscribe, post_post, put_post, post_post_delete, post_post_report +from app.api.alpha.utils.post import get_post_list, get_post, post_post_like, put_post_save, put_post_subscribe, post_post, put_post, post_post_delete, post_post_report, post_post_lock from app.api.alpha.utils.reply import get_reply_list, post_reply_like, put_reply_save, put_reply_subscribe, post_reply, put_reply, post_reply_delete, post_reply_report from app.api.alpha.utils.community import get_community, get_community_list, post_community_follow, post_community_block from app.api.alpha.utils.user import get_user, post_user_block diff --git a/app/api/alpha/utils/post.py b/app/api/alpha/utils/post.py index 32deb0ce..d3beabb8 100644 --- a/app/api/alpha/utils/post.py +++ b/app/api/alpha/utils/post.py @@ -3,7 +3,7 @@ from app.api.alpha.views import post_view, post_report_view from app.api.alpha.utils.validators import required, integer_expected, boolean_expected, string_expected from app.constants import POST_TYPE_ARTICLE, POST_TYPE_LINK, POST_TYPE_IMAGE, POST_TYPE_VIDEO from app.models import Post, Community, CommunityMember, utcnow -from app.shared.post import vote_for_post, bookmark_the_post, remove_the_bookmark_from_post, toggle_post_notification, make_post, edit_post, delete_post, restore_post, report_post +from app.shared.post import vote_for_post, bookmark_the_post, remove_the_bookmark_from_post, toggle_post_notification, make_post, edit_post, delete_post, restore_post, report_post, lock_post from app.utils import authorise_api_user, blocked_users, blocked_communities, blocked_instances, community_ids_from_instances, is_image_url, is_video_url from datetime import timedelta @@ -239,3 +239,16 @@ def post_post_report(auth, data): post_json = post_report_view(report=report, post_id=post_id, user_id=user_id) return post_json + +def post_post_lock(auth, data): + required(['post_id', 'locked'], data) + integer_expected(['post_id'], data) + boolean_expected(['locked'], data) + + post_id = data['post_id'] + locked = data['locked'] + + user_id, post = lock_post(post_id, locked, SRC_API, auth) + + post_json = post_view(post=post, variant=4, user_id=user_id) + return post_json diff --git a/app/shared/post.py b/app/shared/post.py index f330a4c5..3f9333fb 100644 --- a/app/shared/post.py +++ b/app/shared/post.py @@ -7,7 +7,7 @@ from app.shared.tasks import task_selector from app.utils import render_template, authorise_api_user, shorten_string, gibberish, ensure_directory_exists, \ piefed_markdown_to_lemmy_markdown, markdown_to_html, remove_tracking_from_link, domain_from_url, \ opengraph_parse, url_to_thumbnail_file, can_create_post, is_video_hosting_site, recently_upvoted_posts, \ - is_image_url, is_video_hosting_site + is_image_url, is_video_hosting_site, add_to_modlog_activitypub from flask import abort, flash, redirect, request, url_for, current_app, g from flask_babel import _ @@ -531,3 +531,35 @@ def report_post(post_id, input, src, auth=None): return user_id, report else: return + + +def lock_post(post_id, locked, src, auth=None): + if src == SRC_API: + user = authorise_api_user(auth, return_type='model') + else: + user = current_user + + post = Post.query.filter_by(id=post_id).one() + if locked: + comments_enabled = False + modlog_type = 'lock_post' + else: + comments_enabled = True + modlog_type = 'unlock_post' + + if post.community.is_moderator(user) or post.community.is_instance_admin(user): + post.comments_enabled = comments_enabled + db.session.commit() + add_to_modlog_activitypub(modlog_type, user, community_id=post.community_id, + link_text=shorten_string(post.title), link=f'post/{post.id}', reason='') + + if locked: + task_selector('lock_post', user_id=user.id, post_id=post_id) + else: + task_selector('unlock_post', user_id=user.id, post_id=post_id) + + return user.id, post + + + + diff --git a/app/shared/tasks/__init__.py b/app/shared/tasks/__init__.py index 9d5bba5a..9e22628d 100644 --- a/app/shared/tasks/__init__.py +++ b/app/shared/tasks/__init__.py @@ -4,6 +4,7 @@ from app.shared.tasks.notes import make_reply, edit_reply from app.shared.tasks.deletes import delete_reply, restore_reply, delete_post, restore_post from app.shared.tasks.flags import report_reply, report_post from app.shared.tasks.pages import make_post, edit_post +from app.shared.tasks.locks import lock_post, unlock_post from flask import current_app @@ -23,7 +24,9 @@ def task_selector(task_key, send_async=True, **kwargs): 'edit_post': edit_post, 'delete_post': delete_post, 'restore_post': restore_post, - 'report_post': report_post + 'report_post': report_post, + 'lock_post': lock_post, + 'unlock_post': unlock_post } if current_app.debug: diff --git a/app/shared/tasks/locks.py b/app/shared/tasks/locks.py new file mode 100644 index 00000000..61f027cf --- /dev/null +++ b/app/shared/tasks/locks.py @@ -0,0 +1,97 @@ +from app import celery +from app.activitypub.signature import default_context, post_request +from app.models import Post, User +from app.utils import gibberish, instance_banned + +from flask import current_app + + +""" JSON format +Lock: +{ + 'id': + 'type': + 'actor': + 'object': + '@context': + 'audience': + 'to': [] + 'cc': [] +} +For Announce, remove @context from inner object, and use same fields except audience +""" + + +@celery.task +def lock_post(send_async, user_id, post_id): + post = Post.query.filter_by(id=post_id).one() + lock_object(user_id, post) + + +@celery.task +def unlock_post(send_async, user_id, post_id): + post = Post.query.filter_by(id=post_id).one() + lock_object(user_id, post, is_undo=True) + + +def lock_object(user_id, object, is_undo=False): + user = User.query.filter_by(id=user_id).one() + community = object.community + + if community.local_only or not community.instance.online(): + return + + lock_id = f"https://{current_app.config['SERVER_NAME']}/activities/lock/{gibberish(15)}" + to = ["https://www.w3.org/ns/activitystreams#Public"] + cc = [community.public_url()] + lock = { + 'id': lock_id, + 'type': 'Lock', + 'actor': user.public_url(), + 'object': object.public_url(), + '@context': default_context(), + 'audience': community.public_url(), + 'to': to, + 'cc': cc + } + + if is_undo: + del lock['@context'] + undo_id = f"https://{current_app.config['SERVER_NAME']}/activities/undo/{gibberish(15)}" + undo = { + 'id': undo_id, + 'type': 'Undo', + 'actor': user.public_url(), + 'object': lock, + '@context': default_context(), + 'audience': community.public_url(), + 'to': to, + 'cc': cc + } + + if community.is_local(): + if is_undo: + del undo['@context'] + object=undo + else: + del lock['@context'] + object=lock + + announce_id = f"https://{current_app.config['SERVER_NAME']}/activities/announce/{gibberish(15)}" + actor = community.public_url() + cc = [community.ap_followers_url] + announce = { + 'id': announce_id, + 'type': 'Announce', + 'actor': actor, + 'object': object, + '@context': default_context(), + 'to': to, + 'cc': cc + } + for instance in community.following_instances(): + if instance.inbox and instance.online() and not user.has_blocked_instance(instance.id) and not instance_banned(instance.domain): + post_request(instance.inbox, announce, community.private_key, community.public_url() + '#main-key') + else: + payload = undo if is_undo else lock + post_request(community.ap_inbox_url, payload, user.private_key, user.public_url() + '#main-key') From 2112bf68290c6d7f5712a1af18874dbb3b8c819b Mon Sep 17 00:00:00 2001 From: freamon Date: Sun, 19 Jan 2025 22:53:32 +0000 Subject: [PATCH 06/16] API: post add/remove sticky --- app/api/alpha/routes.py | 15 +++++- app/api/alpha/utils/__init__.py | 2 +- app/api/alpha/utils/post.py | 17 ++++++- app/shared/post.py | 22 +++++++++ app/shared/tasks/__init__.py | 6 ++- app/shared/tasks/adds.py | 82 +++++++++++++++++++++++++++++++++ app/shared/tasks/removes.py | 82 +++++++++++++++++++++++++++++++++ 7 files changed, 222 insertions(+), 4 deletions(-) create mode 100644 app/shared/tasks/adds.py create mode 100644 app/shared/tasks/removes.py diff --git a/app/api/alpha/routes.py b/app/api/alpha/routes.py index ef03ebe8..c3556ed1 100644 --- a/app/api/alpha/routes.py +++ b/app/api/alpha/routes.py @@ -1,7 +1,7 @@ from app.api.alpha import bp from app.api.alpha.utils import get_site, post_site_block, \ get_search, \ - get_post_list, get_post, post_post_like, put_post_save, put_post_subscribe, post_post, put_post, post_post_delete, post_post_report, post_post_lock, \ + get_post_list, get_post, post_post_like, put_post_save, put_post_subscribe, post_post, put_post, post_post_delete, post_post_report, post_post_lock, post_post_feature, \ get_reply_list, post_reply_like, put_reply_save, put_reply_subscribe, post_reply, put_reply, post_reply_delete, post_reply_report, \ get_community_list, get_community, post_community_follow, post_community_block, \ get_user, post_user_block @@ -218,6 +218,19 @@ def post_alpha_post_lock(): except Exception as ex: return jsonify({"error": str(ex)}), 400 + +@bp.route('/api/alpha/post/feature', methods=['POST']) +def post_alpha_post_feature(): + if not enable_api(): + return jsonify({'error': 'alpha api is not enabled'}) + try: + auth = request.headers.get('Authorization') + data = request.get_json(force=True) or {} + return jsonify(post_post_feature(auth, data)) + except Exception as ex: + return jsonify({"error": str(ex)}), 400 + + # Reply @bp.route('/api/alpha/comment/list', methods=['GET']) def get_alpha_comment_list(): diff --git a/app/api/alpha/utils/__init__.py b/app/api/alpha/utils/__init__.py index 37d627e0..2aa98bbb 100644 --- a/app/api/alpha/utils/__init__.py +++ b/app/api/alpha/utils/__init__.py @@ -1,6 +1,6 @@ from app.api.alpha.utils.site import get_site, post_site_block from app.api.alpha.utils.misc import get_search -from app.api.alpha.utils.post import get_post_list, get_post, post_post_like, put_post_save, put_post_subscribe, post_post, put_post, post_post_delete, post_post_report, post_post_lock +from app.api.alpha.utils.post import get_post_list, get_post, post_post_like, put_post_save, put_post_subscribe, post_post, put_post, post_post_delete, post_post_report, post_post_lock, post_post_feature from app.api.alpha.utils.reply import get_reply_list, post_reply_like, put_reply_save, put_reply_subscribe, post_reply, put_reply, post_reply_delete, post_reply_report from app.api.alpha.utils.community import get_community, get_community_list, post_community_follow, post_community_block from app.api.alpha.utils.user import get_user, post_user_block diff --git a/app/api/alpha/utils/post.py b/app/api/alpha/utils/post.py index d3beabb8..a46cf38a 100644 --- a/app/api/alpha/utils/post.py +++ b/app/api/alpha/utils/post.py @@ -3,7 +3,7 @@ from app.api.alpha.views import post_view, post_report_view from app.api.alpha.utils.validators import required, integer_expected, boolean_expected, string_expected from app.constants import POST_TYPE_ARTICLE, POST_TYPE_LINK, POST_TYPE_IMAGE, POST_TYPE_VIDEO from app.models import Post, Community, CommunityMember, utcnow -from app.shared.post import vote_for_post, bookmark_the_post, remove_the_bookmark_from_post, toggle_post_notification, make_post, edit_post, delete_post, restore_post, report_post, lock_post +from app.shared.post import vote_for_post, bookmark_the_post, remove_the_bookmark_from_post, toggle_post_notification, make_post, edit_post, delete_post, restore_post, report_post, lock_post, sticky_post from app.utils import authorise_api_user, blocked_users, blocked_communities, blocked_instances, community_ids_from_instances, is_image_url, is_video_url from datetime import timedelta @@ -252,3 +252,18 @@ def post_post_lock(auth, data): post_json = post_view(post=post, variant=4, user_id=user_id) return post_json + + +def post_post_feature(auth, data): + required(['post_id', 'featured', 'feature_type'], data) + integer_expected(['post_id'], data) + boolean_expected(['featured'], data) + string_expected(['feature_type'], data) + + post_id = data['post_id'] + featured = data['featured'] + + user_id, post = sticky_post(post_id, featured, SRC_API, auth) + + post_json = post_view(post=post, variant=4, user_id=user_id) + return post_json diff --git a/app/shared/post.py b/app/shared/post.py index 3f9333fb..894861f3 100644 --- a/app/shared/post.py +++ b/app/shared/post.py @@ -561,5 +561,27 @@ def lock_post(post_id, locked, src, auth=None): return user.id, post +def sticky_post(post_id, featured, src, auth=None): + if src == SRC_API: + user = authorise_api_user(auth, return_type='model') + else: + user = current_user + + post = Post.query.filter_by(id=post_id).one() + community = post.community + + if post.community.is_moderator(user) or post.community.is_instance_admin(user): + post.sticky = featured + if not community.ap_featured_url: + community.ap_featured_url = community.ap_profile_id + '/featured' + db.session.commit() + + if featured: + task_selector('sticky_post', user_id=user.id, post_id=post_id) + else: + task_selector('unsticky_post', user_id=user.id, post_id=post_id) + + return user.id, post + diff --git a/app/shared/tasks/__init__.py b/app/shared/tasks/__init__.py index 9e22628d..3684a7b2 100644 --- a/app/shared/tasks/__init__.py +++ b/app/shared/tasks/__init__.py @@ -5,6 +5,8 @@ from app.shared.tasks.deletes import delete_reply, restore_reply, delete_post, r from app.shared.tasks.flags import report_reply, report_post from app.shared.tasks.pages import make_post, edit_post from app.shared.tasks.locks import lock_post, unlock_post +from app.shared.tasks.adds import sticky_post +from app.shared.tasks.removes import unsticky_post from flask import current_app @@ -26,7 +28,9 @@ def task_selector(task_key, send_async=True, **kwargs): 'restore_post': restore_post, 'report_post': report_post, 'lock_post': lock_post, - 'unlock_post': unlock_post + 'unlock_post': unlock_post, + 'sticky_post': sticky_post, + 'unsticky_post': unsticky_post } if current_app.debug: diff --git a/app/shared/tasks/adds.py b/app/shared/tasks/adds.py new file mode 100644 index 00000000..3a259cc0 --- /dev/null +++ b/app/shared/tasks/adds.py @@ -0,0 +1,82 @@ +from app import celery +from app.activitypub.signature import default_context, post_request +from app.models import Community, Post, User +from app.utils import gibberish, instance_banned + +from flask import current_app + + +""" JSON format +Add: +{ + 'id': + 'type': + 'actor': + 'object': + 'target': (featured_url or moderators_url) + '@context': + 'audience': + 'to': [] + 'cc': [] +} +For Announce, remove @context from inner object, and use same fields except audience +""" + + +@celery.task +def sticky_post(send_async, user_id, post_id): + post = Post.query.filter_by(id=post_id).one() + add_object(user_id, post) + + +@celery.task +def add_mod(send_async, user_id, mod_id, community_id): + mod = User.query.filter_by(id=mod_id).one() + add_object(user_id, mod, community_id) + + +def add_object(user_id, object, community_id=None): + user = User.query.filter_by(id=user_id).one() + if not community_id: + community = object.community + else: + community = Community.query.filter_by(id=community_id).one() + + if community.local_only or not community.instance.online(): + return + + add_id = f"https://{current_app.config['SERVER_NAME']}/activities/add/{gibberish(15)}" + to = ["https://www.w3.org/ns/activitystreams#Public"] + cc = [community.public_url()] + add = { + 'id': add_id, + 'type': 'Add', + 'actor': user.public_url(), + 'object': object.public_url(), + 'target': community.ap_moderators_url if community_id else community.ap_featured_url, + '@context': default_context(), + 'audience': community.public_url(), + 'to': to, + 'cc': cc + } + + if community.is_local(): + del add['@context'] + + announce_id = f"https://{current_app.config['SERVER_NAME']}/activities/announce/{gibberish(15)}" + actor = community.public_url() + cc = [community.ap_followers_url] + announce = { + 'id': announce_id, + 'type': 'Announce', + 'actor': actor, + 'object': add, + '@context': default_context(), + 'to': to, + 'cc': cc + } + for instance in community.following_instances(): + if instance.inbox and instance.online() and not user.has_blocked_instance(instance.id) and not instance_banned(instance.domain): + post_request(instance.inbox, announce, community.private_key, community.public_url() + '#main-key') + else: + post_request(community.ap_inbox_url, add, user.private_key, user.public_url() + '#main-key') diff --git a/app/shared/tasks/removes.py b/app/shared/tasks/removes.py new file mode 100644 index 00000000..99f09fec --- /dev/null +++ b/app/shared/tasks/removes.py @@ -0,0 +1,82 @@ +from app import celery +from app.activitypub.signature import default_context, post_request +from app.models import Community, Post, User +from app.utils import gibberish, instance_banned + +from flask import current_app + + +""" JSON format +Remove: +{ + 'id': + 'type': + 'actor': + 'object': + 'target': (featured_url or moderators_url) + '@context': + 'audience': + 'to': [] + 'cc': [] +} +For Announce, remove @context from inner object, and use same fields except audience +""" + + +@celery.task +def unsticky_post(send_async, user_id, post_id): + post = Post.query.filter_by(id=post_id).one() + remove_object(user_id, post) + + +@celery.task +def remove_mod(send_async, user_id, mod_id, community_id): + mod = User.query.filter_by(id=mod_id).one() + remove_object(user_id, mod, community_id) + + +def remove_object(user_id, object, community_id=None): + user = User.query.filter_by(id=user_id).one() + if not community_id: + community = object.community + else: + community = Community.query.filter_by(id=community_id).one() + + if community.local_only or not community.instance.online(): + return + + remove_id = f"https://{current_app.config['SERVER_NAME']}/activities/remove/{gibberish(15)}" + to = ["https://www.w3.org/ns/activitystreams#Public"] + cc = [community.public_url()] + remove = { + 'id': remove_id, + 'type': 'Remove', + 'actor': user.public_url(), + 'object': object.public_url(), + 'target': community.ap_moderators_url if community_id else community.ap_featured_url, + '@context': default_context(), + 'audience': community.public_url(), + 'to': to, + 'cc': cc + } + + if community.is_local(): + del remove['@context'] + + announce_id = f"https://{current_app.config['SERVER_NAME']}/activities/announce/{gibberish(15)}" + actor = community.public_url() + cc = [community.ap_followers_url] + announce = { + 'id': announce_id, + 'type': 'Announce', + 'actor': actor, + 'object': remove, + '@context': default_context(), + 'to': to, + 'cc': cc + } + for instance in community.following_instances(): + if instance.inbox and instance.online() and not user.has_blocked_instance(instance.id) and not instance_banned(instance.domain): + post_request(instance.inbox, announce, community.private_key, community.public_url() + '#main-key') + else: + post_request(community.ap_inbox_url, remove, user.private_key, user.public_url() + '#main-key') From ac53c2635b955f6cf61dd3ed1d68daee279c9c98 Mon Sep 17 00:00:00 2001 From: freamon Date: Mon, 20 Jan 2025 03:02:46 +0000 Subject: [PATCH 07/16] Fixes for manual post retrieval --- app/activitypub/util.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/app/activitypub/util.py b/app/activitypub/util.py index 608f1946..119d0186 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -2291,7 +2291,7 @@ def resolve_remote_post(uri: str, community, announce_id, store_ap_json) -> Unio # called from UI, via 'search' option in navbar, or 'Retrieve a post from the original server' in community sidebar def resolve_remote_post_from_search(uri: str) -> Union[Post, None]: - post = Post.query.filter_by(ap_id=uri).first() + post = Post.get_by_ap_id(uri) if post: return post @@ -2335,7 +2335,7 @@ def resolve_remote_post_from_search(uri: str) -> Union[Post, None]: return None # check again that it doesn't already exist (can happen with different but equivalent URLs) - post = Post.query.filter_by(ap_id=post_data['id']).first() + post = Post.get_by_ap_id(post_data['id']) if post: return post @@ -2367,7 +2367,7 @@ def resolve_remote_post_from_search(uri: str) -> Union[Post, None]: # find the post's author user = find_actor_or_create(actor) if user and community and post_data: - request_json = {'id': f"https://{uri_domain}/activities/create/gibberish(15)", 'object': post_data} + request_json = {'id': f"https://{uri_domain}/activities/create/{gibberish(15)}", 'object': post_data} post = create_post(False, community, request_json, user) if post: if 'published' in post_data: @@ -2508,6 +2508,8 @@ def log_incoming_ap(id, aplog_type, aplog_result, saved_json, message=None): def find_community(request_json): + # Create/Update from platform that included Community in 'audience', 'cc', or 'to' in outer or inner object + # Also works for manually retrieved posts locations = ['audience', 'cc', 'to'] if 'object' in request_json and isinstance(request_json['object'], dict): rjs = [request_json, request_json['object']] @@ -2529,9 +2531,20 @@ def find_community(request_json): if potential_community: return potential_community + # used for manual retrieval of a PeerTube vid + if request_json['type'] == 'Video': + if 'attributedTo' in request_json and isinstance(request_json['attributedTo'], list): + for a in request_json['attributedTo']: + if a['type'] == 'Group': + potential_community = Community.query.filter_by(ap_profile_id=a['id'].lower()).first() + if potential_community: + return potential_community + + # change this if manual retrieval of comments is allowed in future if not 'object' in request_json: return None + # Create/Update Note from platform that didn't include the Community in 'audience', 'cc', or 'to' (e.g. Mastodon reply to Lemmy post) if 'inReplyTo' in request_json['object'] and request_json['object']['inReplyTo'] is not None: post_being_replied_to = Post.query.filter_by(ap_id=request_json['object']['inReplyTo'].lower()).first() if post_being_replied_to: @@ -2541,7 +2554,8 @@ def find_community(request_json): if comment_being_replied_to: return comment_being_replied_to.community - if request_json['object']['type'] == 'Video': # PeerTube + # Update / Video from PeerTube (possibly an edit, more likely an invite to query Likes / Replies endpoints) + if request_json['object']['type'] == 'Video': if 'attributedTo' in request_json['object'] and isinstance(request_json['object']['attributedTo'], list): for a in request_json['object']['attributedTo']: if a['type'] == 'Group': From 113a64a95d3b8cec316d4f189cd02680a18acb1f Mon Sep 17 00:00:00 2001 From: freamon Date: Mon, 20 Jan 2025 05:01:31 +0000 Subject: [PATCH 08/16] API: post remove and restore by mod --- app/api/alpha/routes.py | 153 +++++++++++++++++--------------- app/api/alpha/utils/__init__.py | 2 +- app/api/alpha/utils/post.py | 23 ++++- app/shared/post.py | 70 +++++++++++++++ app/shared/tasks/deletes.py | 20 +++-- 5 files changed, 188 insertions(+), 80 deletions(-) diff --git a/app/api/alpha/routes.py b/app/api/alpha/routes.py index c3556ed1..e1d30dc3 100644 --- a/app/api/alpha/routes.py +++ b/app/api/alpha/routes.py @@ -1,8 +1,10 @@ from app.api.alpha import bp from app.api.alpha.utils import get_site, post_site_block, \ get_search, \ - get_post_list, get_post, post_post_like, put_post_save, put_post_subscribe, post_post, put_post, post_post_delete, post_post_report, post_post_lock, post_post_feature, \ - get_reply_list, post_reply_like, put_reply_save, put_reply_subscribe, post_reply, put_reply, post_reply_delete, post_reply_report, \ + get_post_list, get_post, post_post_like, put_post_save, put_post_subscribe, post_post, \ + put_post, post_post_delete, post_post_report, post_post_lock, post_post_feature, post_post_remove, \ + get_reply_list, post_reply_like, put_reply_save, put_reply_subscribe, post_reply, put_reply, \ + post_reply_delete, post_reply_report, \ get_community_list, get_community, post_community_follow, post_community_block, \ get_user, post_user_block from app.shared.auth import log_user_in @@ -231,6 +233,18 @@ def post_alpha_post_feature(): return jsonify({"error": str(ex)}), 400 +@bp.route('/api/alpha/post/remove', methods=['POST']) +def post_alpha_post_remove(): + if not enable_api(): + return jsonify({'error': 'alpha api is not enabled'}) + #try: + auth = request.headers.get('Authorization') + data = request.get_json(force=True) or {} + return jsonify(post_post_remove(auth, data)) + #except Exception as ex: + # return jsonify({"error": str(ex)}), 400 + + # Reply @bp.route('/api/alpha/comment/list', methods=['GET']) def get_alpha_comment_list(): @@ -368,108 +382,105 @@ def post_alpha_user_block(): # Not yet implemented. Copied from lemmy's V3 api, so some aren't needed, and some need changing # Site - not yet implemented -@bp.route('/api/alpha/site', methods=['POST']) -@bp.route('/api/alpha/site', methods=['PUT']) +@bp.route('/api/alpha/site', methods=['POST']) # Create New Site. No plans to implement +@bp.route('/api/alpha/site', methods=['PUT']) # Edit Site. Not available in app def alpha_site(): return jsonify({"error": "not_yet_implemented"}), 400 # Miscellaneous - not yet implemented -@bp.route('/api/alpha/modlog', methods=['GET']) -@bp.route('/api/alpha/resolve_object', methods=['GET']) -@bp.route('/api/alpha/federated_instances', methods=['GET']) +@bp.route('/api/alpha/modlog', methods=['GET']) # Get Modlog. Not usually public +@bp.route('/api/alpha/resolve_object', methods=['GET']) # Stage 1: Needed for search +@bp.route('/api/alpha/federated_instances', methods=['GET']) # No plans to implement - only V3 version needed def alpha_miscellaneous(): return jsonify({"error": "not_yet_implemented"}), 400 # Community - not yet implemented -@bp.route('/api/alpha/community', methods=['POST']) -@bp.route('/api/alpha/community', methods=['PUT']) -@bp.route('/api/alpha/community/hide', methods=['PUT']) -@bp.route('/api/alpha/community/delete', methods=['POST']) -@bp.route('/api/alpha/community/remove', methods=['POST']) -@bp.route('/api/alpha/community/transfer', methods=['POST']) -@bp.route('/api/alpha/community/ban_user', methods=['POST']) -@bp.route('/api/alpha/community/mod', methods=['POST']) +@bp.route('/api/alpha/community', methods=['POST']) # (none +@bp.route('/api/alpha/community', methods=['PUT']) # of +@bp.route('/api/alpha/community/hide', methods=['PUT']) # these +@bp.route('/api/alpha/community/delete', methods=['POST']) # are +@bp.route('/api/alpha/community/remove', methods=['POST']) # available +@bp.route('/api/alpha/community/transfer', methods=['POST']) # in +@bp.route('/api/alpha/community/ban_user', methods=['POST']) # the +@bp.route('/api/alpha/community/mod', methods=['POST']) # app) def alpha_community(): return jsonify({"error": "not_yet_implemented"}), 400 # Post - not yet implemented -@bp.route('/api/alpha/post/remove', methods=['POST']) -@bp.route('/api/alpha/post/feature', methods=['POST']) -@bp.route('/api/alpha/post/report', methods=['POST']) -@bp.route('/api/alpha/post/report/resolve', methods=['PUT']) -@bp.route('/api/alpha/post/report/list', methods=['GET']) -@bp.route('/api/alpha/post/site_metadata', methods=['GET']) +@bp.route('/api/alpha/post/report/resolve', methods=['PUT']) # Stage 2 +@bp.route('/api/alpha/post/report/list', methods=['GET']) # Stage 2 +@bp.route('/api/alpha/post/site_metadata', methods=['GET']) # Not available in app def alpha_post(): return jsonify({"error": "not_yet_implemented"}), 400 # Reply - not yet implemented -@bp.route('/api/alpha/comment', methods=['GET']) -@bp.route('/api/alpha/comment/remove', methods=['POST']) -@bp.route('/api/alpha/comment/mark_as_read', methods=['POST']) -@bp.route('/api/alpha/comment/distinguish', methods=['POST']) -@bp.route('/api/alpha/comment/report/resolve', methods=['PUT']) -@bp.route('/api/alpha/comment/report/list', methods=['GET']) +@bp.route('/api/alpha/comment', methods=['GET']) # Stage 1 if needed for search +@bp.route('/api/alpha/comment/remove', methods=['POST']) # Stage 1 +@bp.route('/api/alpha/comment/mark_as_read', methods=['POST']) # No DB support +@bp.route('/api/alpha/comment/distinguish', methods=['POST']) # Not really used +@bp.route('/api/alpha/comment/report/resolve', methods=['PUT']) # Stage 2 +@bp.route('/api/alpha/comment/report/list', methods=['GET']) # Stage 2 def alpha_reply(): return jsonify({"error": "not_yet_implemented"}), 400 # Chat - not yet implemented -@bp.route('/api/alpha/private_message/list', methods=['GET']) -@bp.route('/api/alpha/private_message', methods=['PUT']) -@bp.route('/api/alpha/private_message', methods=['POST']) -@bp.route('/api/alpha/private_message/delete', methods=['POST']) -@bp.route('/api/alpha/private_message/mark_as_read', methods=['POST']) -@bp.route('/api/alpha/private_message/report', methods=['POST']) -@bp.route('/api/alpha/private_message/report/resolve', methods=['PUT']) -@bp.route('/api/alpha/private_message/report/list', methods=['GET']) +@bp.route('/api/alpha/private_message/list', methods=['GET']) # Stage 1 +@bp.route('/api/alpha/private_message', methods=['PUT']) # Stage 1 +@bp.route('/api/alpha/private_message', methods=['POST']) # Stage 1 +@bp.route('/api/alpha/private_message/delete', methods=['POST']) # Stage 1 +@bp.route('/api/alpha/private_message/mark_as_read', methods=['POST']) # Stage 1 +@bp.route('/api/alpha/private_message/report', methods=['POST']) # Stage 1 +@bp.route('/api/alpha/private_message/report/resolve', methods=['PUT']) # Stage 2 +@bp.route('/api/alpha/private_message/report/list', methods=['GET']) # Stage 2 def alpha_chat(): return jsonify({"error": "not_yet_implemented"}), 400 # User - not yet implemented -@bp.route('/api/alpha/user/register', methods=['POST']) -@bp.route('/api/alpha/user/get_captcha', methods=['GET']) -@bp.route('/api/alpha/user/mention', methods=['GET']) -@bp.route('/api/alpha/user/mention/mark_as_read', methods=['POST']) -@bp.route('/api/alpha/user/replies', methods=['GET']) -@bp.route('/api/alpha/user/ban', methods=['POST']) -@bp.route('/api/alpha/user/banned', methods=['GET']) -@bp.route('/api/alpha/user/delete_account', methods=['POST']) -@bp.route('/api/alpha/user/password_reset', methods=['POST']) -@bp.route('/api/alpha/user/password_change', methods=['POST']) -@bp.route('/api/alpha/user/mark_all_as_read', methods=['POST']) -@bp.route('/api/alpha/user/save_user_settings', methods=['PUT']) -@bp.route('/api/alpha/user/change_password', methods=['PUT']) -@bp.route('/api/alpha/user/repost_count', methods=['GET']) -@bp.route('/api/alpha/user/unread_count', methods=['GET']) -@bp.route('/api/alpha/user/verify_email', methods=['POST']) -@bp.route('/api/alpha/user/leave_admin', methods=['POST']) -@bp.route('/api/alpha/user/totp/generate', methods=['POST']) -@bp.route('/api/alpha/user/totp/update', methods=['POST']) -@bp.route('/api/alpha/user/export_settings', methods=['GET']) -@bp.route('/api/alpha/user/import_settings', methods=['POST']) -@bp.route('/api/alpha/user/list_logins', methods=['GET']) -@bp.route('/api/alpha/user/validate_auth', methods=['GET']) -@bp.route('/api/alpha/user/logout', methods=['POST']) +@bp.route('/api/alpha/user/register', methods=['POST']) # Not available in app +@bp.route('/api/alpha/user/get_captcha', methods=['GET']) # Not available in app +@bp.route('/api/alpha/user/mention', methods=['GET']) # No DB support +@bp.route('/api/alpha/user/mention/mark_as_read', methods=['POST']) # No DB support +@bp.route('/api/alpha/user/replies', methods=['GET']) # Stage 1 +@bp.route('/api/alpha/user/ban', methods=['POST']) # Admin function. No plans to implement +@bp.route('/api/alpha/user/banned', methods=['GET']) # Admin function. No plans to implement +@bp.route('/api/alpha/user/delete_account', methods=['POST']) # Not available in app +@bp.route('/api/alpha/user/password_reset', methods=['POST']) # Not available in app +@bp.route('/api/alpha/user/password_change', methods=['POST']) # Not available in app +@bp.route('/api/alpha/user/mark_all_as_read', methods=['POST']) # Stage 1 +@bp.route('/api/alpha/user/save_user_settings', methods=['PUT']) # Not available in app +@bp.route('/api/alpha/user/change_password', methods=['PUT']) # Not available in app +@bp.route('/api/alpha/user/report_count', methods=['GET']) # Stage 2 +@bp.route('/api/alpha/user/unread_count', methods=['GET']) # Stage 1 +@bp.route('/api/alpha/user/verify_email', methods=['POST']) # Admin function. No plans to implement +@bp.route('/api/alpha/user/leave_admin', methods=['POST']) # Admin function. No plans to implement +@bp.route('/api/alpha/user/totp/generate', methods=['POST']) # Not available in app +@bp.route('/api/alpha/user/totp/update', methods=['POST']) # Not available in app +@bp.route('/api/alpha/user/export_settings', methods=['GET']) # Not available in app +@bp.route('/api/alpha/user/import_settings', methods=['POST']) # Not available in app +@bp.route('/api/alpha/user/list_logins', methods=['GET']) # Not available in app +@bp.route('/api/alpha/user/validate_auth', methods=['GET']) # Not available in app +@bp.route('/api/alpha/user/logout', methods=['POST']) # Stage 1 def alpha_user(): return jsonify({"error": "not_yet_implemented"}), 400 # Admin - not yet implemented @bp.route('/api/alpha/admin/add', methods=['POST']) -@bp.route('/api/alpha/admin/registration_application/count', methods=['GET']) -@bp.route('/api/alpha/admin/registration_application/list', methods=['GET']) -@bp.route('/api/alpha/admin/registration_application/approve', methods=['PUT']) -@bp.route('/api/alpha/admin/purge/person', methods=['POST']) -@bp.route('/api/alpha/admin/purge/community', methods=['POST']) -@bp.route('/api/alpha/admin/purge/post', methods=['POST']) -@bp.route('/api/alpha/admin/purge/comment', methods=['POST']) -@bp.route('/api/alpha/post/like/list', methods=['GET']) -@bp.route('/api/alpha/comment/like/list', methods=['GET']) +@bp.route('/api/alpha/admin/registration_application/count', methods=['GET']) # (no +@bp.route('/api/alpha/admin/registration_application/list', methods=['GET']) # plans +@bp.route('/api/alpha/admin/registration_application/approve', methods=['PUT']) # to +@bp.route('/api/alpha/admin/purge/person', methods=['POST']) # implement +@bp.route('/api/alpha/admin/purge/community', methods=['POST']) # any +@bp.route('/api/alpha/admin/purge/post', methods=['POST']) # endpoints +@bp.route('/api/alpha/admin/purge/comment', methods=['POST']) # for +@bp.route('/api/alpha/post/like/list', methods=['GET']) # admin +@bp.route('/api/alpha/comment/like/list', methods=['GET']) # use) def alpha_admin(): return jsonify({"error": "not_yet_implemented"}), 400 # CustomEmoji - not yet implemented -@bp.route('/api/alpha/custom_emoji', methods=['PUT']) -@bp.route('/api/alpha/custom_emoji', methods=['POST']) -@bp.route('/api/alpha/custom_emoji/delete', methods=['POST']) +@bp.route('/api/alpha/custom_emoji', methods=['PUT']) # (doesn't +@bp.route('/api/alpha/custom_emoji', methods=['POST']) # seem +@bp.route('/api/alpha/custom_emoji/delete', methods=['POST']) # important) def alpha_emoji(): return jsonify({"error": "not_yet_implemented"}), 400 diff --git a/app/api/alpha/utils/__init__.py b/app/api/alpha/utils/__init__.py index 2aa98bbb..944e8164 100644 --- a/app/api/alpha/utils/__init__.py +++ b/app/api/alpha/utils/__init__.py @@ -1,6 +1,6 @@ from app.api.alpha.utils.site import get_site, post_site_block from app.api.alpha.utils.misc import get_search -from app.api.alpha.utils.post import get_post_list, get_post, post_post_like, put_post_save, put_post_subscribe, post_post, put_post, post_post_delete, post_post_report, post_post_lock, post_post_feature +from app.api.alpha.utils.post import get_post_list, get_post, post_post_like, put_post_save, put_post_subscribe, post_post, put_post, post_post_delete, post_post_report, post_post_lock, post_post_feature, post_post_remove from app.api.alpha.utils.reply import get_reply_list, post_reply_like, put_reply_save, put_reply_subscribe, post_reply, put_reply, post_reply_delete, post_reply_report from app.api.alpha.utils.community import get_community, get_community_list, post_community_follow, post_community_block from app.api.alpha.utils.user import get_user, post_user_block diff --git a/app/api/alpha/utils/post.py b/app/api/alpha/utils/post.py index a46cf38a..f1902c92 100644 --- a/app/api/alpha/utils/post.py +++ b/app/api/alpha/utils/post.py @@ -3,7 +3,8 @@ from app.api.alpha.views import post_view, post_report_view from app.api.alpha.utils.validators import required, integer_expected, boolean_expected, string_expected from app.constants import POST_TYPE_ARTICLE, POST_TYPE_LINK, POST_TYPE_IMAGE, POST_TYPE_VIDEO from app.models import Post, Community, CommunityMember, utcnow -from app.shared.post import vote_for_post, bookmark_the_post, remove_the_bookmark_from_post, toggle_post_notification, make_post, edit_post, delete_post, restore_post, report_post, lock_post, sticky_post +from app.shared.post import vote_for_post, bookmark_the_post, remove_the_bookmark_from_post, toggle_post_notification, make_post, edit_post, \ + delete_post, restore_post, report_post, lock_post, sticky_post, mod_remove_post, mod_restore_post from app.utils import authorise_api_user, blocked_users, blocked_communities, blocked_instances, community_ids_from_instances, is_image_url, is_video_url from datetime import timedelta @@ -267,3 +268,23 @@ def post_post_feature(auth, data): post_json = post_view(post=post, variant=4, user_id=user_id) return post_json + + +def post_post_remove(auth, data): + required(['post_id', 'removed'], data) + integer_expected(['post_id'], data) + boolean_expected(['removed'], data) + string_expected(['reason'], data) + + post_id = data['post_id'] + removed = data['removed'] + + if removed == True: + reason = data['reason'] if 'reason' in data else 'Removed by mod' + user_id, post = mod_remove_post(post_id, reason, SRC_API, auth) + else: + reason = data['reason'] if 'reason' in data else 'Restored by mod' + user_id, post = mod_restore_post(post_id, reason, SRC_API, auth) + + post_json = post_view(post=post, variant=4, user_id=user_id) + return post_json diff --git a/app/shared/post.py b/app/shared/post.py index 894861f3..67846b28 100644 --- a/app/shared/post.py +++ b/app/shared/post.py @@ -572,9 +572,15 @@ def sticky_post(post_id, featured, src, auth=None): if post.community.is_moderator(user) or post.community.is_instance_admin(user): post.sticky = featured + if featured: + modlog_type = 'featured_post' + else: + modlog_type = 'unfeatured_post' if not community.ap_featured_url: community.ap_featured_url = community.ap_profile_id + '/featured' db.session.commit() + add_to_modlog_activitypub(modlog_type, user, community_id=post.community_id, + link_text=shorten_string(post.title), link=f'post/{post.id}', reason='') if featured: task_selector('sticky_post', user_id=user.id, post_id=post_id) @@ -584,4 +590,68 @@ def sticky_post(post_id, featured, src, auth=None): return user.id, post +# mod deletes +def mod_remove_post(post_id, reason, src, auth): + if src == SRC_API: + user = authorise_api_user(auth, return_type='model') + else: + user = current_user + + post = Post.query.filter_by(id=post_id, user_id=user.id, deleted=False).one() + if not post.community.is_moderator(user) and not post.community.is_instance_admin(user): + raise Exception('Does not have permission') + + if post.url: + post.calculate_cross_posts(delete_only=True) + + post.deleted = True + post.deleted_by = user.id + post.author.post_count -= 1 + post.community.post_count -= 1 + db.session.commit() + if src == SRC_WEB: + flash(_('Post deleted.')) + + add_to_modlog_activitypub('delete_post', user, community_id=post.community_id, + link_text=shorten_string(post.title), link=f'post/{post.id}', reason=reason) + + task_selector('delete_post', user_id=user.id, post_id=post.id, reason=reason) + + if src == SRC_API: + return user.id, post + else: + return + + +def mod_restore_post(post_id, reason, src, auth): + if src == SRC_API: + user = authorise_api_user(auth, return_type='model') + else: + user = current_user + + post = Post.query.filter_by(id=post_id, user_id=user.id, deleted=True).one() + if not post.community.is_moderator(user) and not post.community.is_instance_admin(user): + raise Exception('Does not have permission') + + if post.url: + post.calculate_cross_posts() + + post.deleted = False + post.deleted_by = None + post.author.post_count -= 1 + post.community.post_count -= 1 + db.session.commit() + if src == SRC_WEB: + flash(_('Post restored.')) + + add_to_modlog_activitypub('restore_post', user, community_id=post.community_id, + link_text=shorten_string(post.title), link=f'post/{post.id}', reason=reason) + + task_selector('restore_post', user_id=user.id, post_id=post.id, reason=reason) + + if src == SRC_API: + return user.id, post + else: + return + diff --git a/app/shared/tasks/deletes.py b/app/shared/tasks/deletes.py index d82902b7..6d0aa0f6 100644 --- a/app/shared/tasks/deletes.py +++ b/app/shared/tasks/deletes.py @@ -13,6 +13,7 @@ Delete: 'type': 'actor': 'object': + 'summary': (if deleted by mod / admin) '@context': 'audience': 'to': [] @@ -23,30 +24,30 @@ For Announce, remove @context from inner object, and use same fields except audi @celery.task -def delete_reply(send_async, user_id, reply_id): +def delete_reply(send_async, user_id, reply_id, reason=None): reply = PostReply.query.filter_by(id=reply_id).one() delete_object(user_id, reply) @celery.task -def restore_reply(send_async, user_id, reply_id): +def restore_reply(send_async, user_id, reply_id, reason=None): reply = PostReply.query.filter_by(id=reply_id).one() delete_object(user_id, reply, is_restore=True) @celery.task -def delete_post(send_async, user_id, post_id): +def delete_post(send_async, user_id, post_id, reason=None): post = Post.query.filter_by(id=post_id).one() - delete_object(user_id, post, is_post=True) + delete_object(user_id, post, is_post=True, reason=reason) @celery.task -def restore_post(send_async, user_id, post_id): +def restore_post(send_async, user_id, post_id, reason=None): post = Post.query.filter_by(id=post_id).one() - delete_object(user_id, post, is_post=True, is_restore=True) + delete_object(user_id, post, is_post=True, is_restore=True, reason=reason) -def delete_object(user_id, object, is_post=False, is_restore=False): +def delete_object(user_id, object, is_post=False, is_restore=False, reason=None): user = User.query.filter_by(id=user_id).one() community = object.community @@ -81,6 +82,8 @@ def delete_object(user_id, object, is_post=False, is_restore=False): 'to': to, 'cc': cc } + if reason: + delete['summary'] = reason if is_restore: del delete['@context'] @@ -127,6 +130,9 @@ def delete_object(user_id, object, is_post=False, is_restore=False): post_request(community.ap_inbox_url, payload, user.private_key, user.public_url() + '#main-key') domains_sent_to.append(community.instance.domain) + if reason: + return + if is_post and followers: payload = undo if is_restore else delete for follower in followers: From 7cfef851205378c797eb6bb28a2cf06b508ee185 Mon Sep 17 00:00:00 2001 From: freamon Date: Mon, 20 Jan 2025 05:49:47 +0000 Subject: [PATCH 09/16] API: reply remove and restore by mod --- app/api/alpha/routes.py | 27 ++++++++++---- app/api/alpha/utils/__init__.py | 2 +- app/api/alpha/utils/reply.py | 21 ++++++++++- app/shared/reply.py | 65 ++++++++++++++++++++++++++++++++- app/shared/tasks/deletes.py | 4 +- 5 files changed, 106 insertions(+), 13 deletions(-) diff --git a/app/api/alpha/routes.py b/app/api/alpha/routes.py index e1d30dc3..5eccf6e8 100644 --- a/app/api/alpha/routes.py +++ b/app/api/alpha/routes.py @@ -4,7 +4,7 @@ from app.api.alpha.utils import get_site, post_site_block, \ get_post_list, get_post, post_post_like, put_post_save, put_post_subscribe, post_post, \ put_post, post_post_delete, post_post_report, post_post_lock, post_post_feature, post_post_remove, \ get_reply_list, post_reply_like, put_reply_save, put_reply_subscribe, post_reply, put_reply, \ - post_reply_delete, post_reply_report, \ + post_reply_delete, post_reply_report, post_reply_remove, \ get_community_list, get_community, post_community_follow, post_community_block, \ get_user, post_user_block from app.shared.auth import log_user_in @@ -237,12 +237,12 @@ def post_alpha_post_feature(): def post_alpha_post_remove(): if not enable_api(): return jsonify({'error': 'alpha api is not enabled'}) - #try: - auth = request.headers.get('Authorization') - data = request.get_json(force=True) or {} - return jsonify(post_post_remove(auth, data)) - #except Exception as ex: - # return jsonify({"error": str(ex)}), 400 + try: + auth = request.headers.get('Authorization') + data = request.get_json(force=True) or {} + return jsonify(post_post_remove(auth, data)) + except Exception as ex: + return jsonify({"error": str(ex)}), 400 # Reply @@ -342,6 +342,18 @@ def post_alpha_comment_report(): return jsonify({"error": str(ex)}), 400 +@bp.route('/api/alpha/comment/remove', methods=['POST']) +def post_alpha_comment_remove(): + if not enable_api(): + return jsonify({'error': 'alpha api is not enabled'}) + try: + auth = request.headers.get('Authorization') + data = request.get_json(force=True) or {} + return jsonify(post_reply_remove(auth, data)) + except Exception as ex: + return jsonify({"error": str(ex)}), 400 + + # User @bp.route('/api/alpha/user', methods=['GET']) def get_alpha_user(): @@ -415,7 +427,6 @@ def alpha_post(): # Reply - not yet implemented @bp.route('/api/alpha/comment', methods=['GET']) # Stage 1 if needed for search -@bp.route('/api/alpha/comment/remove', methods=['POST']) # Stage 1 @bp.route('/api/alpha/comment/mark_as_read', methods=['POST']) # No DB support @bp.route('/api/alpha/comment/distinguish', methods=['POST']) # Not really used @bp.route('/api/alpha/comment/report/resolve', methods=['PUT']) # Stage 2 diff --git a/app/api/alpha/utils/__init__.py b/app/api/alpha/utils/__init__.py index 944e8164..8b015d27 100644 --- a/app/api/alpha/utils/__init__.py +++ b/app/api/alpha/utils/__init__.py @@ -1,7 +1,7 @@ from app.api.alpha.utils.site import get_site, post_site_block from app.api.alpha.utils.misc import get_search from app.api.alpha.utils.post import get_post_list, get_post, post_post_like, put_post_save, put_post_subscribe, post_post, put_post, post_post_delete, post_post_report, post_post_lock, post_post_feature, post_post_remove -from app.api.alpha.utils.reply import get_reply_list, post_reply_like, put_reply_save, put_reply_subscribe, post_reply, put_reply, post_reply_delete, post_reply_report +from app.api.alpha.utils.reply import get_reply_list, post_reply_like, put_reply_save, put_reply_subscribe, post_reply, put_reply, post_reply_delete, post_reply_report, post_reply_remove from app.api.alpha.utils.community import get_community, get_community_list, post_community_follow, post_community_block from app.api.alpha.utils.user import get_user, post_user_block diff --git a/app/api/alpha/utils/reply.py b/app/api/alpha/utils/reply.py index 436b29fb..ea598269 100644 --- a/app/api/alpha/utils/reply.py +++ b/app/api/alpha/utils/reply.py @@ -3,7 +3,7 @@ from app.api.alpha.utils.validators import required, integer_expected, boolean_e from app.api.alpha.views import reply_view, reply_report_view from app.models import PostReply, Post from app.shared.reply import vote_for_reply, bookmark_the_post_reply, remove_the_bookmark_from_post_reply, toggle_post_reply_notification, make_reply, edit_reply, \ - delete_reply, restore_reply, report_reply + delete_reply, restore_reply, report_reply, mod_remove_reply, mod_restore_reply from app.utils import authorise_api_user, blocked_users, blocked_instances from sqlalchemy import desc @@ -191,3 +191,22 @@ def post_reply_report(auth, data): reply_json = reply_report_view(report=report, reply_id=reply_id, user_id=user_id) return reply_json + +def post_reply_remove(auth, data): + required(['comment_id', 'removed'], data) + integer_expected(['comment_id'], data) + boolean_expected(['removed'], data) + string_expected(['reason'], data) + + reply_id = data['comment_id'] + removed = data['removed'] + + if removed == True: + reason = data['reason'] if 'reason' in data else 'Removed by mod' + user_id, reply = mod_remove_reply(reply_id, reason, SRC_API, auth) + else: + reason = data['reason'] if 'reason' in data else 'Restored by mod' + user_id, reply = mod_restore_reply(reply_id, reason, SRC_API, auth) + + reply_json = reply_view(reply=reply, variant=4, user_id=user_id) + return reply_json diff --git a/app/shared/reply.py b/app/shared/reply.py index 8e6c0560..7470fad9 100644 --- a/app/shared/reply.py +++ b/app/shared/reply.py @@ -5,7 +5,7 @@ from app.constants import * from app.models import Instance, Notification, NotificationSubscription, Post, PostReply, PostReplyBookmark, Report, Site, User, utcnow from app.shared.tasks import task_selector from app.utils import gibberish, instance_banned, render_template, authorise_api_user, recently_upvoted_post_replies, recently_downvoted_post_replies, shorten_string, \ - piefed_markdown_to_lemmy_markdown, markdown_to_html, ap_datetime + piefed_markdown_to_lemmy_markdown, markdown_to_html, ap_datetime, add_to_modlog_activitypub from flask import abort, current_app, flash, redirect, request, url_for from flask_babel import _ @@ -337,3 +337,66 @@ def report_reply(reply_id, input, src, auth=None): return user_id, report else: return + + +# mod deletes +def mod_remove_reply(reply_id, reason, src, auth): + if src == SRC_API: + user = authorise_api_user(auth, return_type='model') + else: + user = current_user + + reply = PostReply.query.filter_by(id=reply_id, deleted=False).one() + if not reply.community.is_moderator(user) and not reply.community.is_instance_admin(user): + raise Exception('Does not have permission') + + reply.deleted = True + reply.deleted_by = user.id + if not reply.author.bot: + reply.post.reply_count -= 1 + reply.author.post_reply_count -= 1 + db.session.commit() + if src == SRC_WEB: + flash(_('Comment deleted.')) + + add_to_modlog_activitypub('delete_post_reply', user, community_id=reply.community_id, + link_text=shorten_string(f'comment on {shorten_string(reply.post.title)}'), + link=f'post/{reply.post_id}#comment_{reply.id}', reason=reason) + + task_selector('delete_reply', user_id=user.id, reply_id=reply.id, reason=reason) + + if src == SRC_API: + return user.id, reply + else: + return + + +def mod_restore_reply(reply_id, reason, src, auth): + if src == SRC_API: + user = authorise_api_user(auth, return_type='model') + else: + user = current_user + + reply = PostReply.query.filter_by(id=reply_id, deleted=True).one() + if not reply.community.is_moderator(user) and not reply.community.is_instance_admin(user): + raise Exception('Does not have permission') + + reply.deleted = False + reply.deleted_by = None + if not reply.author.bot: + reply.post.reply_count += 1 + reply.author.post_reply_count += 1 + db.session.commit() + if src == SRC_WEB: + flash(_('Comment restored.')) + + add_to_modlog_activitypub('restore_post_reply', user, community_id=reply.community_id, + link_text=shorten_string(f'comment on {shorten_string(reply.post.title)}'), + link=f'post/{reply.post_id}#comment_{reply.id}', reason=reason) + + task_selector('restore_reply', user_id=user.id, reply_id=reply.id, reason=reason) + + if src == SRC_API: + return user.id, reply + else: + return diff --git a/app/shared/tasks/deletes.py b/app/shared/tasks/deletes.py index 6d0aa0f6..9cf5f7f3 100644 --- a/app/shared/tasks/deletes.py +++ b/app/shared/tasks/deletes.py @@ -26,13 +26,13 @@ For Announce, remove @context from inner object, and use same fields except audi @celery.task def delete_reply(send_async, user_id, reply_id, reason=None): reply = PostReply.query.filter_by(id=reply_id).one() - delete_object(user_id, reply) + delete_object(user_id, reply, reason=reason) @celery.task def restore_reply(send_async, user_id, reply_id, reason=None): reply = PostReply.query.filter_by(id=reply_id).one() - delete_object(user_id, reply, is_restore=True) + delete_object(user_id, reply, is_restore=True, reason=reason) @celery.task From f1fe8f07f1576f2598df0bb67e2813106d111401 Mon Sep 17 00:00:00 2001 From: freamon Date: Mon, 20 Jan 2025 13:14:18 +0000 Subject: [PATCH 10/16] Treat updates Announced by a.gup.pe / peertube as new if no local version exists --- app/activitypub/util.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/app/activitypub/util.py b/app/activitypub/util.py index 119d0186..1f0d0146 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -2260,7 +2260,9 @@ def resolve_remote_post(uri: str, community, announce_id, store_ap_json) -> Unio post_reply = PostReply.get_by_ap_id(uri) if post_reply: update_post_reply_from_activity(post_reply, request_json) - else: + else: + activity = 'create' + if activity == 'create': post_reply = create_post_reply(store_ap_json, community, request_json['object']['inReplyTo'], request_json, user) if post_reply: if 'published' in post_data: @@ -2274,8 +2276,10 @@ def resolve_remote_post(uri: str, community, announce_id, store_ap_json) -> Unio if activity == 'update': post = Post.get_by_ap_id(uri) if post: - update_post_from_activity(post_reply, request_json) - else: + update_post_from_activity(post, request_json) + else: + activity = 'create' + if activity == 'create': post = create_post(store_ap_json, community, request_json, user, announce_id) if post: if 'published' in post_data: From dbc8dc4c4a4694904dd19ddd61c3397cdc8d2b54 Mon Sep 17 00:00:00 2001 From: freamon Date: Mon, 20 Jan 2025 15:09:22 +0000 Subject: [PATCH 11/16] API: support app's 'Trending Communities' screen --- app/api/alpha/utils/community.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/app/api/alpha/utils/community.py b/app/api/alpha/utils/community.py index 8222b377..ca3a3a7c 100644 --- a/app/api/alpha/utils/community.py +++ b/app/api/alpha/utils/community.py @@ -6,33 +6,46 @@ from app.models import Community, CommunityMember from app.shared.community import join_community, leave_community, block_community, unblock_community from app.utils import communities_banned_from, blocked_instances +from sqlalchemy import desc + @cache.memoize(timeout=3) -def cached_community_list(type, user_id): - if type == 'Subscribed' and user_id is not None: - communities = Community.query.filter_by(banned=False).join(CommunityMember).filter(CommunityMember.user_id == user_id) +def cached_community_list(type, sort, limit, user_id): + if user_id: banned_from = communities_banned_from(user_id) - if banned_from: - communities = communities.filter(Community.id.not_in(banned_from)) + else: + banned_from = None + + if type == 'Subscribed': + communities = Community.query.filter_by(banned=False).join(CommunityMember).filter(CommunityMember.user_id == user_id) + elif type == 'Local': + communities = Community.query.filter_by(ap_id=None, banned=False) else: communities = Community.query.filter_by(banned=False) - if user_id is not None: + if banned_from: + communities = communities.filter(Community.id.not_in(banned_from)) + + if user_id: blocked_instance_ids = blocked_instances(user_id) if blocked_instance_ids: communities = communities.filter(Community.instance_id.not_in(blocked_instance_ids)) + if sort == 'Active': # 'Trending Communities' screen + communities = communities.order_by(desc(Community.last_active)).limit(limit) + return communities.all() def get_community_list(auth, data): type = data['type_'] if data and 'type_' in data else "All" + sort = data['sort'] if data and 'sort' in data else "Hot" page = int(data['page']) if data and 'page' in data else 1 limit = int(data['limit']) if data and 'limit' in data else 10 user_id = authorise_api_user(auth) if auth else None - communities = cached_community_list(type, user_id) + communities = cached_community_list(type, sort, limit, user_id) start = (page - 1) * limit end = start + limit From 4071e9208bd7abdd0e395eb15ffefef0262cf825 Mon Sep 17 00:00:00 2001 From: freamon Date: Mon, 20 Jan 2025 23:40:55 +0000 Subject: [PATCH 12/16] Support manual retrieval of posts from nodebb --- app/activitypub/util.py | 83 +++++++++++++++++++++++++---------------- 1 file changed, 50 insertions(+), 33 deletions(-) diff --git a/app/activitypub/util.py b/app/activitypub/util.py index 1f0d0146..30ccf8e1 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -2181,6 +2181,45 @@ def can_delete(user_ap_id, post): return can_edit(user_ap_id, post) +# TODO: import this into community/util for backfilling, instead of having 2 copies, and - also - call it from resolve_remote_post() +def remote_object_to_json(uri): + try: + object_request = get_request(uri, headers={'Accept': 'application/activity+json'}) + except httpx.HTTPError: + time.sleep(3) + try: + object_request = get_request(uri, headers={'Accept': 'application/activity+json'}) + except httpx.HTTPError: + return None + if object_request.status_code == 200: + try: + object = object_request.json() + return object + except: + object_request.close() + return None + object_request.close() + elif object_request.status_code == 401: + try: + site = Site.query.get(1) + object_request = signed_get_request(uri, site.private_key, f"https://{current_app.config['SERVER_NAME']}/actor#main-key") + except httpx.HTTPError: + time.sleep(3) + try: + object_request = signed_get_request(uri, site.private_key, f"https://{current_app.config['SERVER_NAME']}/actor#main-key") + except httpx.HTTPError: + return None + try: + object = object_request.json() + return object + except: + object_request.close() + return None + object_request.close() + else: + return None + + # called from incoming activitypub, when the object in an Announce is just a URL # despite the name, it works for both posts and replies def resolve_remote_post(uri: str, community, announce_id, store_ap_json) -> Union[Post, PostReply, None]: @@ -2304,39 +2343,17 @@ def resolve_remote_post_from_search(uri: str) -> Union[Post, None]: actor_domain = None actor = None - try: - object_request = get_request(uri, headers={'Accept': 'application/activity+json'}) - except httpx.HTTPError: - time.sleep(3) - try: - object_request = get_request(uri, headers={'Accept': 'application/activity+json'}) - except httpx.HTTPError: - return None - if object_request.status_code == 200: - try: - post_data = object_request.json() - except: - object_request.close() - return None - object_request.close() - elif object_request.status_code == 401: - try: - site = Site.query.get(1) - object_request = signed_get_request(uri, site.private_key, f"https://{current_app.config['SERVER_NAME']}/actor#main-key") - except httpx.HTTPError: - time.sleep(3) - try: - object_request = signed_get_request(uri, site.private_key, f"https://{current_app.config['SERVER_NAME']}/actor#main-key") - except httpx.HTTPError: - return None - try: - post_data = object_request.json() - except: - object_request.close() - return None - object_request.close() - else: - return None + post_data = remote_object_to_json(uri) + + # nodebb. the post is the first entry in orderedItems, and the replies are the remaining entries + # just gets orderedItems[0] the retrieve the post + if ('type' in post_data and post_data['type'] == 'OrderedCollection' and + 'totalItems' in post_data and post_data['totalItems'] > 0 and + 'orderedItems' in post_data and isinstance(post_data['orderedItems'], list)): + uri = post_data['orderedItems'][0] + parsed_url = urlparse(uri) + uri_domain = parsed_url.netloc + post_data = remote_object_to_json(uri) # check again that it doesn't already exist (can happen with different but equivalent URLs) post = Post.get_by_ap_id(post_data['id']) From d3fc652f1f6384b89b8474db9fd4a0f9b857cfcc Mon Sep 17 00:00:00 2001 From: freamon Date: Tue, 21 Jan 2025 02:46:51 +0000 Subject: [PATCH 13/16] Truncate UserExtraField labels to fit in DB column --- app/activitypub/util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/activitypub/util.py b/app/activitypub/util.py index 30ccf8e1..218aab46 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -521,7 +521,7 @@ def refresh_user_profile_task(user_id): if field_data['type'] == 'PropertyValue': if ' Date: Tue, 21 Jan 2025 02:48:02 +0000 Subject: [PATCH 14/16] get 10 replies in background to manually retrieved nodebb topics --- app/activitypub/util.py | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/app/activitypub/util.py b/app/activitypub/util.py index 218aab46..8c0b78d8 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -2222,13 +2222,13 @@ def remote_object_to_json(uri): # called from incoming activitypub, when the object in an Announce is just a URL # despite the name, it works for both posts and replies -def resolve_remote_post(uri: str, community, announce_id, store_ap_json) -> Union[Post, PostReply, None]: +def resolve_remote_post(uri: str, community, announce_id, store_ap_json, nodebb=False) -> Union[Post, PostReply, None]: parsed_url = urlparse(uri) uri_domain = parsed_url.netloc announce_actor = community.ap_profile_id parsed_url = urlparse(announce_actor) announce_actor_domain = parsed_url.netloc - if announce_actor_domain != 'a.gup.pe' and announce_actor_domain != uri_domain: + if announce_actor_domain != 'a.gup.pe' and not nodebb and announce_actor_domain != uri_domain: return None actor_domain = None actor = None @@ -2332,6 +2332,20 @@ def resolve_remote_post(uri: str, community, announce_id, store_ap_json) -> Unio return None +@celery.task +def get_nodebb_replies_in_background(replies_uri_list, community_id): + max = 10 if not current_app.debug else 2 # magic number alert + community = Community.query.get(community_id) + if not community: + return + reply_count = 0 + for uri in replies_uri_list: + reply_count += 1 + post_reply = resolve_remote_post(uri, community, None, False, nodebb=True) + if reply_count >= max: + break + + # called from UI, via 'search' option in navbar, or 'Retrieve a post from the original server' in community sidebar def resolve_remote_post_from_search(uri: str) -> Union[Post, None]: post = Post.get_by_ap_id(uri) @@ -2345,11 +2359,14 @@ def resolve_remote_post_from_search(uri: str) -> Union[Post, None]: post_data = remote_object_to_json(uri) - # nodebb. the post is the first entry in orderedItems, and the replies are the remaining entries - # just gets orderedItems[0] the retrieve the post + # nodebb. the post is the first entry in orderedItems of a topic, and the replies are the remaining entries + # just gets orderedItems[0] to retrieve the post, and then replies are retrieved in the background + topic_post_data = post_data + nodebb = False if ('type' in post_data and post_data['type'] == 'OrderedCollection' and 'totalItems' in post_data and post_data['totalItems'] > 0 and 'orderedItems' in post_data and isinstance(post_data['orderedItems'], list)): + nodebb = True uri = post_data['orderedItems'][0] parsed_url = urlparse(uri) uri_domain = parsed_url.netloc @@ -2385,6 +2402,8 @@ def resolve_remote_post_from_search(uri: str) -> Union[Post, None]: # find the community the post was submitted to community = find_community(post_data) + if not community and nodebb: + community = find_community(topic_post_data) # use 'audience' from topic if post has no info for how it got there # find the post's author user = find_actor_or_create(actor) if user and community and post_data: @@ -2395,6 +2414,11 @@ def resolve_remote_post_from_search(uri: str) -> Union[Post, None]: post.posted_at=post_data['published'] post.last_active=post_data['published'] db.session.commit() + if nodebb and topic_post_data['totalItems'] > 1: + if current_app.debug: + get_nodebb_replies_in_background(topic_post_data['orderedItems'][1:], community.id) + else: + get_nodebb_replies_in_background.delay(topic_post_data['orderedItems'][1:], community.id) return post return None From 28030547674478221f44e3826d290de22a14fa6b Mon Sep 17 00:00:00 2001 From: freamon Date: Tue, 21 Jan 2025 03:04:20 +0000 Subject: [PATCH 15/16] Share remote_object_to_json() --- app/activitypub/util.py | 35 +---------------------------------- app/community/util.py | 40 +--------------------------------------- 2 files changed, 2 insertions(+), 73 deletions(-) diff --git a/app/activitypub/util.py b/app/activitypub/util.py index 8c0b78d8..47794cc2 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -2181,7 +2181,6 @@ def can_delete(user_ap_id, post): return can_edit(user_ap_id, post) -# TODO: import this into community/util for backfilling, instead of having 2 copies, and - also - call it from resolve_remote_post() def remote_object_to_json(uri): try: object_request = get_request(uri, headers={'Accept': 'application/activity+json'}) @@ -2233,39 +2232,7 @@ def resolve_remote_post(uri: str, community, announce_id, store_ap_json, nodebb= actor_domain = None actor = None - try: - object_request = get_request(uri, headers={'Accept': 'application/activity+json'}) - except httpx.HTTPError: - time.sleep(3) - try: - object_request = get_request(uri, headers={'Accept': 'application/activity+json'}) - except httpx.HTTPError: - return None - if object_request.status_code == 200: - try: - post_data = object_request.json() - except: - object_request.close() - return None - object_request.close() - elif object_request.status_code == 401: - try: - site = Site.query.get(1) - object_request = signed_get_request(uri, site.private_key, f"https://{current_app.config['SERVER_NAME']}/actor#main-key") - except httpx.HTTPError: - time.sleep(3) - try: - object_request = signed_get_request(uri, site.private_key, f"https://{current_app.config['SERVER_NAME']}/actor#main-key") - except httpx.HTTPError: - return None - try: - post_data = object_request.json() - except: - object_request.close() - return None - object_request.close() - else: - return None + post_data = remote_object_to_json(uri) # find the author. Make sure their domain matches the site hosting it to mitigate impersonation attempts if 'attributedTo' in post_data: diff --git a/app/community/util.py b/app/community/util.py index c2a3dba8..f190f90f 100644 --- a/app/community/util.py +++ b/app/community/util.py @@ -12,7 +12,7 @@ from pillow_heif import register_heif_opener from app import db, cache, celery from app.activitypub.signature import post_request, default_context, signed_get_request from app.activitypub.util import find_actor_or_create, actor_json_to_model, ensure_domains_match, \ - find_hashtag_or_create, create_post + find_hashtag_or_create, create_post, remote_object_to_json from app.constants import POST_TYPE_ARTICLE, POST_TYPE_LINK, POST_TYPE_IMAGE, POST_TYPE_VIDEO, NOTIF_POST, \ POST_TYPE_POLL from app.models import Community, File, BannedInstances, PostReply, Post, utcnow, CommunityMember, Site, \ @@ -79,44 +79,6 @@ def search_for_community(address: str): return None -def remote_object_to_json(uri): - try: - object_request = get_request(uri, headers={'Accept': 'application/activity+json'}) - except httpx.HTTPError: - time.sleep(3) - try: - object_request = get_request(uri, headers={'Accept': 'application/activity+json'}) - except httpx.HTTPError: - return None - if object_request.status_code == 200: - try: - object = object_request.json() - return object - except: - object_request.close() - return None - object_request.close() - elif object_request.status_code == 401: - try: - site = Site.query.get(1) - object_request = signed_get_request(uri, site.private_key, f"https://{current_app.config['SERVER_NAME']}/actor#main-key") - except httpx.HTTPError: - time.sleep(3) - try: - object_request = signed_get_request(uri, site.private_key, f"https://{current_app.config['SERVER_NAME']}/actor#main-key") - except httpx.HTTPError: - return None - try: - object = object_request.json() - return object - except: - object_request.close() - return None - object_request.close() - else: - return None - - @celery.task def retrieve_mods_and_backfill(community_id: int, server, name, community_json=None): with current_app.app_context(): From c0ba8ba1e8415062a46075839043e8709c97836f Mon Sep 17 00:00:00 2001 From: freamon Date: Tue, 21 Jan 2025 06:29:31 +0000 Subject: [PATCH 16/16] Hopefully temporary fix for nodebb bug --- app/activitypub/routes.py | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index 8d2845e6..cbd2c608 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -415,16 +415,19 @@ def shared_inbox(): return '', 200 id = request_json['id'] + missing_actor_in_announce_object = False # nodebb if request_json['type'] == 'Announce' and isinstance(request_json['object'], dict): object = request_json['object'] - if not 'id' in object or not 'type' in object or not 'actor' in object or not 'object' in object: + if not 'actor' in object: + missing_actor_in_announce_object = True + if not 'id' in object or not 'type' in object or not 'object' in object: if 'type' in object and (object['type'] == 'Page' or object['type'] == 'Note'): log_incoming_ap(id, APLOG_ANNOUNCE, APLOG_IGNORED, saved_json, 'Intended for Mastodon') else: log_incoming_ap(id, APLOG_ANNOUNCE, APLOG_FAILURE, saved_json, 'Missing minimum expected fields in JSON Announce object') return '', 200 - if isinstance(object['actor'], str) and object['actor'].startswith('https://' + current_app.config['SERVER_NAME']): + if not missing_actor_in_announce_object and isinstance(object['actor'], str) and object['actor'].startswith('https://' + current_app.config['SERVER_NAME']): log_incoming_ap(id, APLOG_DUPLICATE, APLOG_IGNORED, saved_json, 'Activity about local content which is already present') return '', 200 @@ -499,6 +502,11 @@ def shared_inbox(): process_delete_request.delay(request_json, store_ap_json) return '' + if missing_actor_in_announce_object: + if ((request_json['object']['type'] == 'Create' or request_json['object']['type']) and + 'attributedTo' in request_json['object']['object'] and isinstance(request_json['object']['object']['attributedTo'], str)): + request_json['object']['actor'] = request_json['object']['object']['attributedTo'] + if current_app.debug: process_inbox_request(request_json, store_ap_json) else: @@ -528,17 +536,20 @@ def replay_inbox_request(request_json): return id = request_json['id'] + missing_actor_in_announce_object = False # nodebb if request_json['type'] == 'Announce' and isinstance(request_json['object'], dict): object = request_json['object'] - if not 'id' in object or not 'type' in object or not 'actor' in object or not 'object' in object: + if not 'actor' in object: + missing_actor_in_announce_object = True + if not 'id' in object or not 'type' in object or not 'object' in object: if 'type' in object and (object['type'] == 'Page' or object['type'] == 'Note'): log_incoming_ap(id, APLOG_ANNOUNCE, APLOG_IGNORED, request_json, 'REPLAY: Intended for Mastodon') else: log_incoming_ap(id, APLOG_ANNOUNCE, APLOG_FAILURE, request_json, 'REPLAY: Missing minimum expected fields in JSON Announce object') return - if isinstance(object['actor'], str) and object['actor'].startswith('https://' + current_app.config['SERVER_NAME']): - log_incoming_ap(id, APLOG_DUPLICATE, APLOG_IGNORED, saved_json, 'Activity about local content which is already present') + if not missing_actor_in_announce_object and isinstance(object['actor'], str) and object['actor'].startswith('https://' + current_app.config['SERVER_NAME']): + log_incoming_ap(id, APLOG_DUPLICATE, APLOG_IGNORED, request_json, 'REPLAY: Activity about local content which is already present') return # Ignore unutilised PeerTube activity @@ -573,6 +584,11 @@ def replay_inbox_request(request_json): process_delete_request(request_json, True) return + if missing_actor_in_announce_object: + if ((request_json['object']['type'] == 'Create' or request_json['object']['type']) and + 'attributedTo' in request_json['object']['object'] and isinstance(request_json['object']['object']['attributedTo'], str)): + request_json['object']['actor'] = request_json['object']['object']['attributedTo'] + process_inbox_request(request_json, True) return @@ -799,7 +815,8 @@ def process_inbox_request(request_json, store_ap_json): # inner object of Create is not a ChatMessage else: if (core_activity['object']['type'] == 'Note' and 'name' in core_activity['object'] and # Poll Votes - 'inReplyTo' in core_activity['object'] and 'attributedTo' in core_activity['object']): + 'inReplyTo' in core_activity['object'] and 'attributedTo' in core_activity['object'] and + not 'published' in core_activity['object']): post_being_replied_to = Post.query.filter_by(ap_id=core_activity['object']['inReplyTo']).first() if post_being_replied_to: poll_data = Poll.query.get(post_being_replied_to.id)