diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index a976f170..50216c00 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -21,7 +21,7 @@ from app.activitypub.util import public_key, users_total, active_half_year, acti upvote_post, delete_post_or_comment, community_members, \ user_removed_from_remote_server, create_post, create_post_reply, update_post_reply_from_activity, \ update_post_from_activity, undo_vote, undo_downvote, post_to_page, get_redis_connection, find_reported_object, \ - process_report, ensure_domains_match, can_edit, can_delete, remove_data_from_banned_user + process_report, ensure_domains_match, can_edit, can_delete, remove_data_from_banned_user, resolve_remote_post from app.utils import gibberish, get_setting, is_image_url, allowlist_html, render_template, \ domain_from_url, markdown_to_html, community_membership, ap_datetime, ip_address, can_downvote, \ can_upvote, can_create_post, awaken_dormant_instance, shorten_string, can_create_post_reply, sha256_digest, \ @@ -412,6 +412,14 @@ def shared_inbox(): else: process_delete_request.delay(request_json, activity_log.id, ip_address()) return '' + # Ignore unutilised PeerTube activity + if 'actor' in request_json and request_json['actor'].endswith('accounts/peertube'): + activity_log.result = 'ignored' + activity_log.exception_message = 'PeerTube View or CacheFile activity' + db.session.add(activity_log) + db.session.commit() + return '' + else: activity_log.activity_id = '' if g.site.log_activitypub_json: @@ -595,6 +603,10 @@ def process_inbox_request(request_json, activitypublog_id, ip_address): if isinstance(request_json['object'], str): activity_log.activity_json = json.dumps(request_json) activity_log.exception_message = 'invalid json?' + if 'actor' in request_json: + community = find_actor_or_create(request_json['actor'], community_only=True, create_if_not_found=False) + if community: + resolve_remote_post(request_json['object'], community.id, request_json['actor']) elif request_json['object']['type'] == 'Create': activity_log.activity_type = request_json['object']['type'] if 'object' in request_json and 'object' in request_json['object']: @@ -969,6 +981,16 @@ def process_inbox_request(request_json, activitypublog_id, ip_address): activity_log.exception_message = 'Edit attempt denied' else: activity_log.exception_message = 'PostReply not found' + elif request_json['object']['type'] == 'Video': # PeerTube: editing a video (PT doesn't seem to Announce these) + post = Post.query.filter_by(ap_id=request_json['object']['id']).first() + if post: + if can_edit(request_json['actor'], post): + update_post_from_activity(post, request_json) + activity_log.result = 'success' + else: + activity_log.exception_message = 'Edit attempt denied' + else: + activity_log.exception_message = 'Post not found' elif request_json['type'] == 'Delete': if isinstance(request_json['object'], str): ap_id = request_json['object'] # lemmy @@ -1274,7 +1296,9 @@ def user_inbox(actor): if (('type' in request_json and request_json['type'] == 'Like') or ('type' in request_json and request_json['type'] == 'Undo' and 'object' in request_json and request_json['object']['type'] == 'Like')): - return shared_inbox() + return shared_inbox() + if 'type' in request_json and request_json['type'] == 'Accept': + return shared_inbox() try: HttpSignature.verify_request(request, actor.public_key, skip_date=True) if 'type' in request_json and request_json['type'] == 'Follow': diff --git a/app/activitypub/signature.py b/app/activitypub/signature.py index 1ba5e1db..9aeb9553 100644 --- a/app/activitypub/signature.py +++ b/app/activitypub/signature.py @@ -101,7 +101,7 @@ def post_request(uri: str, body: dict | None, private_key: str, key_id: str, con else: try: result = HttpSignature.signed_request(uri, body, private_key, key_id, content_type, method, timeout) - if result.status_code != 200 and result.status_code != 202: + if result.status_code != 200 and result.status_code != 202 and result.status_code != 204: log.result = 'failure' log.exception_message += f' Response status code was {result.status_code}' current_app.logger.error('Response code for post attempt was ' + @@ -109,6 +109,8 @@ def post_request(uri: str, body: dict | None, private_key: str, key_id: str, con log.exception_message += uri if result.status_code == 202: log.exception_message += ' 202' + if result.status_code == 204: + log.exception_message += ' 204' except Exception as e: log.result = 'failure' log.exception_message='could not send:' + str(e) diff --git a/app/activitypub/util.py b/app/activitypub/util.py index 9e137da4..86d87273 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -7,7 +7,7 @@ from random import randint from typing import Union, Tuple import redis -from flask import current_app, request, g, url_for +from flask import current_app, request, g, url_for, json from flask_babel import _ from sqlalchemy import text, func from app import db, cache, constants, celery @@ -488,13 +488,20 @@ def refresh_user_profile_task(user_id): avatar_changed = cover_changed = False if 'icon' in activity_json: - if user.avatar_id and activity_json['icon']['url'] != user.avatar.source_url: - user.avatar.delete_from_disk() - if not user.avatar_id or (user.avatar_id and activity_json['icon']['url'] != user.avatar.source_url): - avatar = File(source_url=activity_json['icon']['url']) - user.avatar = avatar - db.session.add(avatar) - avatar_changed = True + if isinstance(activity_json['icon'], dict) and 'url' in activity_json['icon']: + icon_entry = activity_json['icon']['url'] + elif isinstance(activity_json['icon'], list) and 'url' in activity_json['icon'][-1]: + icon_entry = activity_json['icon'][-1]['url'] + else: + icon_entry = None + if icon_entry: + if user.avatar_id and icon_entry != user.avatar.source_url: + user.avatar.delete_from_disk() + if not user.avatar_id or (user.avatar_id and icon_entry != user.avatar.source_url): + avatar = File(source_url=icon_entry) + user.avatar = avatar + db.session.add(avatar) + avatar_changed = True if 'image' in activity_json: if user.cover_id and activity_json['image']['url'] != user.cover.source_url: user.cover.delete_from_disk() @@ -536,21 +543,22 @@ def refresh_community_profile_task(community_id): activity_json = actor_data.json() actor_data.close() - if 'attributedTo' in activity_json: # lemmy and mbin + if 'attributedTo' in activity_json and isinstance(activity_json['attributedTo'], str): # lemmy and mbin mods_url = activity_json['attributedTo'] elif 'moderators' in activity_json: # kbin mods_url = activity_json['moderators'] else: mods_url = None - community.nsfw = activity_json['sensitive'] + community.nsfw = activity_json['sensitive'] if 'sensitive' in activity_json else False if 'nsfl' in activity_json and activity_json['nsfl']: community.nsfl = activity_json['nsfl'] community.title = activity_json['name'] community.description = activity_json['summary'] if 'summary' in activity_json else '' + community.description_html = markdown_to_html(community.description) community.rules = activity_json['rules'] if 'rules' in activity_json else '' community.rules_html = lemmy_markdown_to_html(activity_json['rules'] if 'rules' in activity_json else '') - community.restricted_to_mods = activity_json['postingRestrictedToMods'] + community.restricted_to_mods = activity_json['postingRestrictedToMods'] if 'postingRestrictedToMods' in activity_json else True community.new_mods_wanted = activity_json['newModsWanted'] if 'newModsWanted' in activity_json else False community.private_mods = activity_json['privateMods'] if 'privateMods' in activity_json else False community.ap_moderators_url = mods_url @@ -567,21 +575,35 @@ def refresh_community_profile_task(community_id): icon_changed = cover_changed = False if 'icon' in activity_json: - if community.icon_id and activity_json['icon']['url'] != community.icon.source_url: - community.icon.delete_from_disk() - if not community.icon_id or (community.icon_id and activity_json['icon']['url'] != community.icon.source_url): - icon = File(source_url=activity_json['icon']['url']) - community.icon = icon - db.session.add(icon) - icon_changed = True + if isinstance(activity_json['icon'], dict) and 'url' in activity_json['icon']: + icon_entry = activity_json['icon']['url'] + elif isinstance(activity_json['icon'], list) and 'url' in activity_json['icon'][-1]: + icon_entry = activity_json['icon'][-1]['url'] + else: + icon_entry = None + if icon_entry: + if community.icon_id and icon_entry != community.icon.source_url: + community.icon.delete_from_disk() + if not community.icon_id or (community.icon_id and icon_entry != community.icon.source_url): + icon = File(source_url=icon_entry) + community.icon = icon + db.session.add(icon) + icon_changed = True if 'image' in activity_json: - if community.image_id and activity_json['image']['url'] != community.image.source_url: - community.image.delete_from_disk() - if not community.image_id or (community.image_id and activity_json['image']['url'] != community.image.source_url): - image = File(source_url=activity_json['image']['url']) - community.image = image - db.session.add(image) - cover_changed = True + if isinstance(activity_json['image'], dict) and 'url' in activity_json['image']: + image_entry = activity_json['image']['url'] + elif isinstance(activity_json['image'], list) and 'url' in activity_json['image'][0]: + image_entry = activity_json['image'][0]['url'] + else: + image_entry = None + if image_entry: + if community.image_id and image_entry != community.image.source_url: + community.image.delete_from_disk() + if not community.image_id or (community.image_id and image_entry != community.image.source_url): + image = File(source_url=image_entry) + community.image = image + db.session.add(image) + cover_changed = True if 'language' in activity_json and isinstance(activity_json['language'], list) and not community.ignore_remote_language: for ap_language in activity_json['language']: new_language = find_language_or_create(ap_language['identifier'], ap_language['name']) @@ -658,10 +680,17 @@ def actor_json_to_model(activity_json, address, server): current_app.logger.error(f'KeyError for {address}@{server} while parsing ' + str(activity_json)) return None - if 'icon' in activity_json and activity_json['icon'] is not None and 'url' in activity_json['icon']: - avatar = File(source_url=activity_json['icon']['url']) - user.avatar = avatar - db.session.add(avatar) + if 'icon' in activity_json and activity_json['icon'] is not None: + if isinstance(activity_json['icon'], dict) and 'url' in activity_json['icon']: + icon_entry = activity_json['icon']['url'] + elif isinstance(activity_json['icon'], list) and 'url' in activity_json['icon'][-1]: + icon_entry = activity_json['icon'][-1]['url'] + else: + icon_entry = None + if icon_entry: + avatar = File(source_url=icon_entry) + user.avatar = avatar + db.session.add(avatar) if 'image' in activity_json and activity_json['image'] is not None and 'url' in activity_json['image']: cover = File(source_url=activity_json['image']['url']) user.cover = cover @@ -674,7 +703,7 @@ def actor_json_to_model(activity_json, address, server): make_image_sizes(user.cover_id, 878, None, 'users') return user elif activity_json['type'] == 'Group': - if 'attributedTo' in activity_json: # lemmy and mbin + if 'attributedTo' in activity_json and isinstance(activity_json['attributedTo'], str): # lemmy and mbin mods_url = activity_json['attributedTo'] elif 'moderators' in activity_json: # kbin mods_url = activity_json['moderators'] @@ -683,7 +712,7 @@ def actor_json_to_model(activity_json, address, server): # only allow nsfw communities if enabled for this instance site = Site.query.get(1) # can't use g.site because actor_json_to_model can be called from celery - if activity_json['sensitive'] and not site.enable_nsfw: + if 'sensitive' in activity_json and activity_json['sensitive'] and not site.enable_nsfw: return None if 'nsfl' in activity_json and activity_json['nsfl'] and not site.enable_nsfl: return None @@ -693,8 +722,8 @@ def actor_json_to_model(activity_json, address, server): description=activity_json['summary'] if 'summary' in activity_json else '', rules=activity_json['rules'] if 'rules' in activity_json else '', rules_html=lemmy_markdown_to_html(activity_json['rules'] if 'rules' in activity_json else ''), - nsfw=activity_json['sensitive'], - restricted_to_mods=activity_json['postingRestrictedToMods'], + nsfw=activity_json['sensitive'] if 'sensitive' in activity_json else False, + restricted_to_mods=activity_json['postingRestrictedToMods'] if 'postingRestrictedToMods' in activity_json else True, new_mods_wanted=activity_json['newModsWanted'] if 'newModsWanted' in activity_json else False, private_mods=activity_json['privateMods'] if 'privateMods' in activity_json else False, created_at=activity_json['published'] if 'published' in activity_json else utcnow(), @@ -714,6 +743,7 @@ def actor_json_to_model(activity_json, address, server): instance_id=find_instance_id(server), low_quality='memes' in activity_json['preferredUsername'] ) + community.description_html = markdown_to_html(community.description) # parse markdown and overwrite html field with result if 'source' in activity_json and \ activity_json['source']['mediaType'] == 'text/markdown': @@ -722,14 +752,28 @@ def actor_json_to_model(activity_json, address, server): elif 'content' in activity_json: community.description_html = allowlist_html(activity_json['content']) community.description = '' - if 'icon' in activity_json and activity_json['icon'] is not None and 'url' in activity_json['icon']: - icon = File(source_url=activity_json['icon']['url']) - community.icon = icon - db.session.add(icon) - if 'image' in activity_json and activity_json['image'] is not None and 'url' in activity_json['image']: - image = File(source_url=activity_json['image']['url']) - community.image = image - db.session.add(image) + if 'icon' in activity_json and activity_json['icon'] is not None: + if isinstance(activity_json['icon'], dict) and 'url' in activity_json['icon']: + icon_entry = activity_json['icon']['url'] + elif isinstance(activity_json['icon'], list) and 'url' in activity_json['icon'][-1]: + icon_entry = activity_json['icon'][-1]['url'] + else: + icon_entry = None + if icon_entry: + icon = File(source_url=icon_entry) + community.icon = icon + db.session.add(icon) + if 'image' in activity_json and activity_json['image'] is not None: + if isinstance(activity_json['image'], dict) and 'url' in activity_json['image']: + image_entry = activity_json['image']['url'] + elif isinstance(activity_json['image'], list) and 'url' in activity_json['image'][0]: + image_entry = activity_json['image'][0]['url'] + else: + image_entry = None + if image_entry: + image = File(source_url=image_entry) + community.image = image + db.session.add(image) if 'language' in activity_json and isinstance(activity_json['language'], list): for ap_language in activity_json['language']: community.languages.append(find_language_or_create(ap_language['identifier'], ap_language['name'])) @@ -763,8 +807,12 @@ def post_json_to_model(activity_log, post_json, user, community) -> Post: post.body = post_json['source']['content'] post.body_html = lemmy_markdown_to_html(post.body) elif 'content' in post_json: - post.body_html = allowlist_html(post_json['content']) - post.body = '' + if post_json['mediaType'] == 'text/html': + post.body_html = allowlist_html(post_json['content']) + post.body = '' + elif post_json['mediaType'] == 'text/markdown': + post.body = post_json['content'] + post.body_html = markdown_to_html(post.body) if 'attachment' in post_json and len(post_json['attachment']) > 0 and 'type' in post_json['attachment'][0]: if post_json['attachment'][0]['type'] == 'Link': post.url = post_json['attachment'][0]['href'] @@ -803,6 +851,14 @@ def post_json_to_model(activity_log, post_json, user, community) -> Post: domain.post_count += 1 post.domain = domain + if post_json['type'] == 'Video': + post.type = POST_TYPE_VIDEO + post.url = post_json['id'] + if 'icon' in post_json and isinstance(post_json['icon'], list): + icon = File(source_url=post_json['icon'][-1]['url']) + db.session.add(icon) + post.image = icon + if 'language' in post_json: language = find_language_or_create(post_json['language']['identifier'], post_json['language']['name']) if language: @@ -1553,8 +1609,12 @@ def create_post(activity_log: ActivityPubLog, community: Community, request_json post.body = request_json['object']['source']['content'] post.body_html = lemmy_markdown_to_html(post.body) elif 'content' in request_json['object'] and request_json['object']['content'] is not None: # Kbin - post.body_html = allowlist_html(request_json['object']['content']) - post.body = '' + if request_json['object']['mediaType'] == 'text/html': + post.body_html = allowlist_html(request_json['object']['content']) + post.body = '' + elif request_json['object']['mediaType'] == 'text/markdown': + post.body = request_json['object']['content'] + post.body_html = markdown_to_html(post.body) if name == "[Microblog]": name += ' ' + microblog_content_to_title(post.body_html) if '[NSFL]' in name.upper() or '(NSFL)' in name.upper(): @@ -1764,8 +1824,12 @@ def update_post_from_activity(post: Post, request_json: dict): post.body = request_json['object']['source']['content'] post.body_html = lemmy_markdown_to_html(post.body) elif 'content' in request_json['object'] and request_json['object']['content'] is not None: # Kbin - post.body_html = allowlist_html(request_json['object']['content']) - post.body = '' + if request_json['object']['mediaType'] == 'text/html': + post.body_html = allowlist_html(request_json['object']['content']) + post.body = '' + elif request_json['object']['mediaType'] == 'text/markdown': + post.body = request_json['object']['content'] + post.body_html = markdown_to_html(post.body) if name == "[Microblog]": name += ' ' + microblog_content_to_title(post.body_html) nsfl_in_title = '[NSFL]' in name.upper() or '(NSFL)' in name.upper() @@ -1778,6 +1842,10 @@ def update_post_from_activity(post: Post, request_json: dict): old_url = post.url old_image_id = post.image_id post.url = '' + if request_json['object']['type'] == 'Video': + post.type = POST_TYPE_VIDEO + # PeerTube URL isn't going to change, so set to old_url to prevent this function changing type or icon + post.url = old_url if 'attachment' in request_json['object'] and len(request_json['object']['attachment']) > 0 and \ 'type' in request_json['object']['attachment'][0]: if request_json['object']['attachment'][0]['type'] == 'Link': @@ -2207,12 +2275,16 @@ def ensure_domains_match(activity: dict) -> bool: else: note_id = None + note_actor = None if 'actor' in activity: note_actor = activity['actor'] - elif 'attributedTo' in activity: + elif 'attributedTo' in activity and isinstance(activity['attributedTo'], str): note_actor = activity['attributedTo'] - else: - note_actor = None + elif 'attributedTo' in activity and isinstance(activity['attributedTo'], list): + for a in activity['attributedTo']: + if a['type'] == 'Person': + note_actor = a['id'] + break if note_id and note_actor: parsed_url = urlparse(note_id) @@ -2238,3 +2310,72 @@ def can_edit(user_ap_id, post): def can_delete(user_ap_id, post): return can_edit(user_ap_id, post) + + +def resolve_remote_post(uri: str, community_id: int, announce_actor=None) -> Union[str, None]: + post = Post.query.filter_by(ap_id=uri).first() + if post: + return post.id + + community = Community.query.get(community_id) + site = Site.query.get(1) + + parsed_url = urlparse(uri) + uri_domain = parsed_url.netloc + if announce_actor: + parsed_url = urlparse(announce_actor) + announce_actor_domain = parsed_url.netloc + if announce_actor_domain != uri_domain: + return None + actor_domain = None + actor = None + post_request = get_request(uri, headers={'Accept': 'application/activity+json'}) + if post_request.status_code == 200: + post_data = post_request.json() + post_request.close() + # check again that it doesn't already exist (can happen with different but equivilent URLs) + post = Post.query.filter_by(ap_id=post_data['id']).first() + if post: + return post.id + if 'attributedTo' in post_data: + if isinstance(post_data['attributedTo'], str): + actor = post_data['attributedTo'] + parsed_url = urlparse(post_data['attributedTo']) + actor_domain = parsed_url.netloc + elif isinstance(post_data['attributedTo'], list): + for a in post_data['attributedTo']: + if a['type'] == 'Person': + actor = a['id'] + parsed_url = urlparse(a['id']) + actor_domain = parsed_url.netloc + break + if uri_domain != actor_domain: + return None + + activity_log = ActivityPubLog(direction='in', activity_id=post_data['id'], activity_type='Resolve Post', result='failure') + if site.log_activitypub_json: + activity_log.activity_json = json.dumps(post_data) + db.session.add(activity_log) + user = find_actor_or_create(actor) + if user and community and post_data: + post = post_json_to_model(activity_log, post_data, user, community) + post.ranking = post_ranking(post.score, post.posted_at) + community.last_active = utcnow() + if post.url: + other_posts = Post.query.filter(Post.id != post.id, Post.url == post.url, + Post.posted_at > post.posted_at - timedelta(days=3), + Post.posted_at < post.posted_at + timedelta(days=3)).all() + for op in other_posts: + if op.cross_posts is None: + op.cross_posts = [post.id] + else: + op.cross_posts.append(post.id) + if post.cross_posts is None: + post.cross_posts = [op.id] + else: + post.cross_posts.append(op.id) + db.session.commit() + if post: + return post.id + + return None diff --git a/app/community/util.py b/app/community/util.py index 16c73c9c..3cd712df 100644 --- a/app/community/util.py +++ b/app/community/util.py @@ -67,6 +67,12 @@ def search_for_community(address: str): if community_json['type'] == 'Group': community = actor_json_to_model(community_json, name, server) if community: + if community.ap_profile_id == f"https://{server}/video-channels/{name}": + if current_app.debug: + retrieve_peertube_mods_and_backfill(community.id, community_json['attributedTo']) + else: + retrieve_peertube_mods_and_backfill.delay(community.id, community_json['attributedTo']) + return community if current_app.debug: retrieve_mods_and_backfill(community.id) else: @@ -75,6 +81,62 @@ def search_for_community(address: str): return None +@celery.task +def retrieve_peertube_mods_and_backfill(community_id: int, mods: list): + community = Community.query.get(community_id) + site = Site.query.get(1) + for m in mods: + user = find_actor_or_create(m['id']) + if user: + existing_membership = CommunityMember.query.filter_by(community_id=community.id, user_id=user.id).first() + if existing_membership: + existing_membership.is_moderator = True + else: + new_membership = CommunityMember(community_id=community.id, user_id=user.id, is_moderator=True) + db.session.add(new_membership) + db.session.commit() + + if community.ap_public_url: + outbox_request = get_request(community.ap_outbox_url, headers={'Accept': 'application/activity+json'}) + if outbox_request.status_code == 200: + outbox_data = outbox_request.json() + outbox_request.close() + if 'totalItems' in outbox_data and outbox_data['totalItems'] > 0: + page1_request = get_request(outbox_data['first'], headers={'Accept': 'application/activity+json'}) + if page1_request.status_code == 200: + page1_data = page1_request.json() + page1_request.close() + if 'type' in page1_data and page1_data['type'] == 'OrderedCollectionPage' and 'orderedItems' in page1_data: + # only 10 posts per page for PeerTube + for activity in page1_data['orderedItems']: + video_request = get_request(activity['object'], headers={'Accept': 'application/activity+json'}) + if video_request.status_code == 200: + video_data = video_request.json() + video_request.close() + activity_log = ActivityPubLog(direction='in', activity_id=video_data['id'], activity_type='Video', result='failure') + if site.log_activitypub_json: + activity_log.activity_json = json.dumps(video_data) + db.session.add(activity_log) + if not ensure_domains_match(video_data): + activity_log.exception_message = 'Domains do not match' + db.session.commit() + continue + if user and user.is_local(): + activity_log.exception_message = 'Activity about local content which is already present' + db.session.commit() + continue + if user: + post = post_json_to_model(activity_log, video_data, user, community) + post.ap_announce_id = activity['id'] + post.ranking = post_ranking(post.score, post.posted_at) + else: + activity_log.exception_message = 'Could not find or create actor' + db.session.commit() + if community.post_count > 0: + community.last_active = Post.query.filter(Post.community_id == community_id).order_by(desc(Post.posted_at)).first().posted_at + db.session.commit() + + @celery.task def retrieve_mods_and_backfill(community_id: int): with current_app.app_context(): diff --git a/app/models.py b/app/models.py index dc2b92ec..7e2d6478 100644 --- a/app/models.py +++ b/app/models.py @@ -1000,6 +1000,10 @@ class Post(db.Model): if vpos != -1: return self.url[vpos + 2:vpos + 13] + def peertube_embed(self): + if self.url: + return self.url.replace('watch', 'embed') + def profile_id(self): if self.ap_id: return self.ap_id diff --git a/app/templates/post/_post_full.html b/app/templates/post/_post_full.html index ae8cb32d..ea5bf961 100644 --- a/app/templates/post/_post_full.html +++ b/app/templates/post/_post_full.html @@ -122,6 +122,9 @@
{% endif %} + {% if 'videos/watch' in post.url %} + + {% endif %} {% elif post.type == POST_TYPE_IMAGE %}