Merge pull request 'Outbox Processing' (#103) from freamon/pyfedi:15 into main

Reviewed-on: https://codeberg.org/rimu/pyfedi/pulls/103

app/activitypub/util.py

@@ -589,21 +589,22 @@ def actor_json_to_model(activity_json, address, server):
         return community
 
 
-def post_json_to_model(post_json, user, community) -> Post:
+def post_json_to_model(activity_log, post_json, user, community) -> Post:
     try:
+        nsfl_in_title = '[NSFL]' in post_json['name'].upper() or '(NSFL)' in post_json['name'].upper()
         post = Post(user_id=user.id, community_id=community.id,
                     title=html.unescape(post_json['name']),
-                    comments_enabled=post_json['commentsEnabled'],
+                    comments_enabled=post_json['commentsEnabled'] if 'commentsEnabled' in post_json else True,
                     sticky=post_json['stickied'] if 'stickied' in post_json else False,
                     nsfw=post_json['sensitive'],
-                    nsfl=post_json['nsfl'] if 'nsfl' in post_json else False,
+                    nsfl=post_json['nsfl'] if 'nsfl' in post_json else nsfl_in_title,
                     ap_id=post_json['id'],
                     type=constants.POST_TYPE_ARTICLE,
                     posted_at=post_json['published'],
                     last_active=post_json['published'],
-                    instance_id=user.instance_id
+                    instance_id=user.instance_id,
+                    indexable = user.indexable
                     )
-        post.indexable = user.indexable
         if 'source' in post_json and \
                 post_json['source']['mediaType'] == 'text/markdown':
             post.body = post_json['source']['content']
@@ -616,8 +617,15 @@ def post_json_to_model(post_json, user, community) -> Post:
                 post.url = post_json['attachment'][0]['href']
                 if is_image_url(post.url):
                     post.type = POST_TYPE_IMAGE
+                    if 'image' in post_json and 'url' in post_json['image']:
+                        image = File(source_url=post_json['image']['url'])
+                    else:
+                        image = File(source_url=post.url)
+                    db.session.add(image)
+                    post.image = image
                 else:
                     post.type = POST_TYPE_LINK
+                    post.url = remove_tracking_from_link(post.url)
 
                 domain = domain_from_url(post.url)
                 # notify about links to banned websites.
@@ -637,10 +645,11 @@ def post_json_to_model(post_json, user, community) -> Post:
                                 admin.unread_notifications += 1
                     if domain.banned:
                         post = None
+                        activity_log.exception_message = domain.name + ' is blocked by admin'
                     if not domain.banned:
                         domain.post_count += 1
                         post.domain = domain
-        if 'image' in post_json and post:
+        if 'image' in post_json and post.image is None:
             image = File(source_url=post_json['image']['url'])
             db.session.add(image)
             post.image = image
@@ -648,7 +657,12 @@ def post_json_to_model(post_json, user, community) -> Post:
         if post is not None:
             db.session.add(post)
             community.post_count += 1
+            activity_log.result = 'success'
             db.session.commit()
+
+            if post.image_id:
+                make_image_sizes(post.image_id, 150, 512, 'posts')  # the 512 sized image is for masonry view
+
         return post
     except KeyError as e:
         current_app.logger.error(f'KeyError in post_json_to_model: ' + str(post_json))

app/community/util.py

@@ -4,7 +4,7 @@ from time import sleep
 from typing import List
 import requests
 from PIL import Image, ImageOps
-from flask import request, abort, g, current_app
+from flask import request, abort, g, current_app, json
 from flask_login import current_user
 from pillow_heif import register_heif_opener
 
@@ -13,11 +13,11 @@ from app.activitypub.signature import post_request
 from app.activitypub.util import find_actor_or_create, actor_json_to_model, post_json_to_model, default_context
 from app.constants import POST_TYPE_ARTICLE, POST_TYPE_LINK, POST_TYPE_IMAGE
 from app.models import Community, File, BannedInstances, PostReply, PostVote, Post, utcnow, CommunityMember, Site, \
-    Instance, Notification, User
+    Instance, Notification, User, ActivityPubLog
 from app.utils import get_request, gibberish, markdown_to_html, domain_from_url, allowlist_html, \
     html_to_markdown, is_image_url, ensure_directory_exists, inbox_domain, post_ranking, shorten_string, parse_page, \
     remove_tracking_from_link, ap_datetime, instance_banned
-from sqlalchemy import func
+from sqlalchemy import func, desc
 import os
 
 
@@ -100,19 +100,26 @@ def retrieve_mods_and_backfill(community_id: int):
                     activities_processed = 0
                     for activity in outbox_data['orderedItems']:
                         user = find_actor_or_create(activity['object']['actor'])
+                        activity_log = ActivityPubLog(direction='in', activity_id=activity['id'], activity_type='Announce', result='failure')
+                        if site.log_activitypub_json:
+                            activity_log.activity_json = json.dumps(activity)
+                        db.session.add(activity_log)
                         if user:
-                            post = post_json_to_model(activity['object']['object'], user, community)
+                            post = post_json_to_model(activity_log, activity['object']['object'], user, community)
                             post.ap_create_id = activity['object']['id']
                             post.ap_announce_id = activity['id']
                             post.ranking = post_ranking(post.score, post.posted_at)
                             db.session.commit()
+                        else:
+                            activity_log.exception_message = 'Could not find or create actor'
+                            db.session.commit()
 
                         activities_processed += 1
                         if activities_processed >= 50:
                             break
                     c = Community.query.get(community.id)
-                    c.post_count = activities_processed
-                    c.last_active = site.last_active = utcnow()
+                    if c.post_count > 0:
+                        c.last_active = Post.query.filter(Post.community_id == community_id).order_by(desc(Post.posted_at)).first().posted_at
                     db.session.commit()
 
 

app/templates/post/_post_full.html

@@ -35,7 +35,7 @@
                         <a href="{{ post.image.view_url() }}" rel="nofollow ugc"><img src="{{ post.image.thumbnail_url() }}" alt="{{ post.image.alt_text if post.image.alt_text else post.title }}"
                              width="{{ post.image.width }}" height="{{ post.image.height }}" /></a>
                     {% else %}
-                        <a href="{{ post.image.view_url() }}" rel="nofollow ugc"><img src="{{ post.image.view_url() }}" alt="{{ post.image.alt_text if post.image.alt_text else post.title }}"
+                        <a href="{{ post.image.view_url() }}" rel="nofollow ugc"><img src="{{ post.image.medium_url() }}" alt="{{ post.image.alt_text if post.image.alt_text else post.title }}"
                              width="{{ post.image.width }}" height="{{ post.image.height }}" /></a>
                     {% endif %}
                 {% else %}

app/templates/post/_post_teaser_masonry.html

@@ -14,7 +14,7 @@
                 {% if post.type == POST_TYPE_LINK %}
                     {% if post.image.medium_url() %}
                         <a href="{{ post.url }}" rel="nofollow ugc" target="_blank" aria-label="{{ _('View image') }}"><img src="{{ post.image.medium_url() }}"
-                           alt="{{ post.image.alt_text if post.image.alt_text else '' }}" loading="lazy" width="{{ post.image.thumbnail_width }}" height="{{ post.image.thumbnail_height }}" /></a>
+                           alt="{{ post.image.alt_text if post.image.alt_text else '' }}" loading="lazy" width="{{ post.image.width }}" height="{{ post.image.height }}" /></a>
                     {% elif post.image.source_url %}
                         <a href="{{ post.url }}" rel="nofollow ugc" target="_blank" aria-label="{{ _('View image') }}"><img src="{{ post.image.source_url }}"
                            alt="{{ post.title }}" loading="lazy" /></a>
@@ -24,7 +24,7 @@
                     {% endif %}
                 {% elif post.type == POST_TYPE_IMAGE %}
                         <a href="{{ post.image.view_url() }}" rel="nofollow ugc" target="_blank"><img src="{{ post.image.medium_url() }}"
-                        alt="{{ post.image.alt_text if post.image.alt_text else '' }}" loading="lazy" width="{{ post.image.thumbnail_width }}" height="{{ post.image.thumbnail_height }}" /></a>
+                        alt="{{ post.image.alt_text if post.image.alt_text else '' }}" loading="lazy" width="{{ post.image.width }}" height="{{ post.image.height }}" /></a>
                 {% else %}
                     <a href="{{ url_for('activitypub.post_ap', post_id=post.id) }}"><img src="{{ post.image.thumbnail_url() }}"
                         alt="{{ post.image.alt_text if post.image.alt_text else '' }}" loading="lazy" /></a>