From c5f0d5175a8f33e356a18644850bad8416192520 Mon Sep 17 00:00:00 2001 From: freamon Date: Fri, 15 Mar 2024 22:32:48 +0000 Subject: [PATCH 1/5] Add ActivityPub logging to outbox processing --- app/activitypub/util.py | 4 +++- app/community/util.py | 13 ++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/app/activitypub/util.py b/app/activitypub/util.py index 425f5125..3c8bd257 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -589,7 +589,7 @@ def actor_json_to_model(activity_json, address, server): return community -def post_json_to_model(post_json, user, community) -> Post: +def post_json_to_model(activity_log, post_json, user, community) -> Post: try: post = Post(user_id=user.id, community_id=community.id, title=html.unescape(post_json['name']), @@ -637,6 +637,7 @@ def post_json_to_model(post_json, user, community) -> Post: admin.unread_notifications += 1 if domain.banned: post = None + activity_log.exception_message = domain.name + ' is blocked by admin' if not domain.banned: domain.post_count += 1 post.domain = domain @@ -648,6 +649,7 @@ def post_json_to_model(post_json, user, community) -> Post: if post is not None: db.session.add(post) community.post_count += 1 + activity_log.result = 'success' db.session.commit() return post except KeyError as e: diff --git a/app/community/util.py b/app/community/util.py index a67b93c9..65411328 100644 --- a/app/community/util.py +++ b/app/community/util.py @@ -4,7 +4,7 @@ from time import sleep from typing import List import requests from PIL import Image, ImageOps -from flask import request, abort, g, current_app +from flask import request, abort, g, current_app, json from flask_login import current_user from pillow_heif import register_heif_opener @@ -13,7 +13,7 @@ from app.activitypub.signature import post_request from app.activitypub.util import find_actor_or_create, actor_json_to_model, post_json_to_model, default_context from app.constants import POST_TYPE_ARTICLE, POST_TYPE_LINK, POST_TYPE_IMAGE from app.models import Community, File, BannedInstances, PostReply, PostVote, Post, utcnow, CommunityMember, Site, \ - Instance, Notification, User + Instance, Notification, User, ActivityPubLog from app.utils import get_request, gibberish, markdown_to_html, domain_from_url, allowlist_html, \ html_to_markdown, is_image_url, ensure_directory_exists, inbox_domain, post_ranking, shorten_string, parse_page, \ remove_tracking_from_link, ap_datetime, instance_banned @@ -100,12 +100,19 @@ def retrieve_mods_and_backfill(community_id: int): activities_processed = 0 for activity in outbox_data['orderedItems']: user = find_actor_or_create(activity['object']['actor']) + activity_log = ActivityPubLog(direction='in', activity_id=activity['id'], activity_type='Announce', result='failure') + if site.log_activitypub_json: + activity_log.activity_json = json.dumps(activity) + db.session.add(activity_log) if user: - post = post_json_to_model(activity['object']['object'], user, community) + post = post_json_to_model(activity_log, activity['object']['object'], user, community) post.ap_create_id = activity['object']['id'] post.ap_announce_id = activity['id'] post.ranking = post_ranking(post.score, post.posted_at) db.session.commit() + else: + activity_log.exception_message = 'Could not find or create actor' + db.session.commit() activities_processed += 1 if activities_processed >= 50: From 98d381d94d456c229f2591f321a18d644e9a2329 Mon Sep 17 00:00:00 2001 From: freamon Date: Fri, 15 Mar 2024 22:51:27 +0000 Subject: [PATCH 2/5] Amend last_active and post_count for newly-discovered remote communities --- app/community/util.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/community/util.py b/app/community/util.py index 65411328..6fc2b438 100644 --- a/app/community/util.py +++ b/app/community/util.py @@ -17,7 +17,7 @@ from app.models import Community, File, BannedInstances, PostReply, PostVote, Po from app.utils import get_request, gibberish, markdown_to_html, domain_from_url, allowlist_html, \ html_to_markdown, is_image_url, ensure_directory_exists, inbox_domain, post_ranking, shorten_string, parse_page, \ remove_tracking_from_link, ap_datetime, instance_banned -from sqlalchemy import func +from sqlalchemy import func, desc import os @@ -118,8 +118,8 @@ def retrieve_mods_and_backfill(community_id: int): if activities_processed >= 50: break c = Community.query.get(community.id) - c.post_count = activities_processed - c.last_active = site.last_active = utcnow() + if c.post_count > 0: + c.last_active = Post.query.filter(Post.community_id == community_id).order_by(desc(Post.posted_at)).first().posted_at db.session.commit() From 37c9431b4ed05aa7299d527e3cd4facf2d4e90b2 Mon Sep 17 00:00:00 2001 From: freamon Date: Fri, 15 Mar 2024 23:47:21 +0000 Subject: [PATCH 3/5] Borrow code from create_post for post_json_to_model --- app/activitypub/util.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/app/activitypub/util.py b/app/activitypub/util.py index 3c8bd257..cbdc2334 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -591,19 +591,20 @@ def actor_json_to_model(activity_json, address, server): def post_json_to_model(activity_log, post_json, user, community) -> Post: try: + nsfl_in_title = '[NSFL]' in post_json['name'].upper() or '(NSFL)' in post_json['name'].upper() post = Post(user_id=user.id, community_id=community.id, title=html.unescape(post_json['name']), - comments_enabled=post_json['commentsEnabled'], + comments_enabled=post_json['commentsEnabled'] if 'commentsEnabled' in post_json else True, sticky=post_json['stickied'] if 'stickied' in post_json else False, nsfw=post_json['sensitive'], - nsfl=post_json['nsfl'] if 'nsfl' in post_json else False, + nsfl=post_json['nsfl'] if 'nsfl' in post_json else nsfl_in_title, ap_id=post_json['id'], type=constants.POST_TYPE_ARTICLE, posted_at=post_json['published'], last_active=post_json['published'], - instance_id=user.instance_id + instance_id=user.instance_id, + indexable = user.indexable ) - post.indexable = user.indexable if 'source' in post_json and \ post_json['source']['mediaType'] == 'text/markdown': post.body = post_json['source']['content'] @@ -616,8 +617,15 @@ def post_json_to_model(activity_log, post_json, user, community) -> Post: post.url = post_json['attachment'][0]['href'] if is_image_url(post.url): post.type = POST_TYPE_IMAGE + if 'image' in post_json and 'url' in post_json['image']: + image = File(source_url=post_json['image']['url']) + else: + image = File(source_url=post.url) + db.session.add(image) + post.image = image else: post.type = POST_TYPE_LINK + post.url = remove_tracking_from_link(post.url) domain = domain_from_url(post.url) # notify about links to banned websites. @@ -641,7 +649,7 @@ def post_json_to_model(activity_log, post_json, user, community) -> Post: if not domain.banned: domain.post_count += 1 post.domain = domain - if 'image' in post_json and post: + if 'image' in post_json and post.image is None: image = File(source_url=post_json['image']['url']) db.session.add(image) post.image = image @@ -651,6 +659,10 @@ def post_json_to_model(activity_log, post_json, user, community) -> Post: community.post_count += 1 activity_log.result = 'success' db.session.commit() + + if post.image_id: + make_image_sizes(post.image_id, 150, 512, 'posts') # the 512 sized image is for masonry view + return post except KeyError as e: current_app.logger.error(f'KeyError in post_json_to_model: ' + str(post_json)) From cbe5f0ca6b8104f8218b882ea67a71e771b28cd4 Mon Sep 17 00:00:00 2001 From: freamon Date: Sat, 16 Mar 2024 00:09:11 +0000 Subject: [PATCH 4/5] Match image size with image source for masonry view --- app/templates/post/_post_teaser_masonry.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/templates/post/_post_teaser_masonry.html b/app/templates/post/_post_teaser_masonry.html index 096add77..7c48629c 100644 --- a/app/templates/post/_post_teaser_masonry.html +++ b/app/templates/post/_post_teaser_masonry.html @@ -14,7 +14,7 @@ {% if post.type == POST_TYPE_LINK %} {% if post.image.medium_url() %} {{ post.image.alt_text if post.image.alt_text else '' }} + alt="{{ post.image.alt_text if post.image.alt_text else '' }}" loading="lazy" width="{{ post.image.width }}" height="{{ post.image.height }}" /> {% elif post.image.source_url %} {{ post.title }} @@ -24,7 +24,7 @@ {% endif %} {% elif post.type == POST_TYPE_IMAGE %} {{ post.image.alt_text if post.image.alt_text else '' }} + alt="{{ post.image.alt_text if post.image.alt_text else '' }}" loading="lazy" width="{{ post.image.width }}" height="{{ post.image.height }}" /> {% else %} {{ post.image.alt_text if post.image.alt_text else '' }} From b57209bb239cfb8064c761e4af2413a6befc3c59 Mon Sep 17 00:00:00 2001 From: freamon Date: Sat, 16 Mar 2024 00:16:47 +0000 Subject: [PATCH 5/5] Serve local image in posts --- app/templates/post/_post_full.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/templates/post/_post_full.html b/app/templates/post/_post_full.html index 0e68f6ca..87828550 100644 --- a/app/templates/post/_post_full.html +++ b/app/templates/post/_post_full.html @@ -35,7 +35,7 @@ {{ post.image.alt_text if post.image.alt_text else post.title }} {% else %} - {{ post.image.alt_text if post.image.alt_text else post.title }}{{ post.image.alt_text if post.image.alt_text else post.title }} {% endif %} {% else %}