Merge pull request 'Follow-up to PR 172: Use signed GET request to fetch remote actor details.' (#180) from freamon/pyfedi:signed_fetch into main

Reviewed-on: https://codeberg.org/rimu/pyfedi/pulls/180

app/activitypub/routes.py

@@ -6,7 +6,7 @@ from app import db, constants, cache, celery
 from app.activitypub import bp
 from flask import request, current_app, abort, jsonify, json, g, url_for, redirect, make_response
 
-from app.activitypub.signature import HttpSignature, post_request, VerificationError
+from app.activitypub.signature import HttpSignature, post_request, VerificationError, default_context
 from app.community.routes import show_community
 from app.community.util import send_to_remote_instance
 from app.post.routes import continue_discussion, show_post
@@ -16,7 +16,7 @@ from app.models import User, Community, CommunityJoinRequest, CommunityMember, C
     PostReply, Instance, PostVote, PostReplyVote, File, AllowedInstances, BannedInstances, utcnow, Site, Notification, \
     ChatMessage, Conversation, UserFollower
 from app.activitypub.util import public_key, users_total, active_half_year, active_month, local_posts, local_comments, \
-    post_to_activity, find_actor_or_create, default_context, instance_blocked, find_reply_parent, find_liked_object, \
+    post_to_activity, find_actor_or_create, instance_blocked, find_reply_parent, find_liked_object, \
     lemmy_site_data, instance_weight, is_activitypub_request, downvote_post_reply, downvote_post, upvote_post_reply, \
     upvote_post, delete_post_or_comment, community_members, \
     user_removed_from_remote_server, create_post, create_post_reply, update_post_reply_from_activity, \
@@ -51,6 +51,28 @@ def webfinger():
         else:
             return 'Webfinger regex failed to match'
 
+        # special case: instance actor
+        if actor == current_app.config['SERVER_NAME']:
+            webfinger_data = {
+              "subject": f"acct:{actor}@{current_app.config['SERVER_NAME']}",
+              "aliases": [f"https://{current_app.config['SERVER_NAME']}/actor"],
+              "links": [
+                {
+                    "rel": "http://webfinger.net/rel/profile-page",
+                    "type": "text/html",
+                    "href": f"https://{current_app.config['SERVER_NAME']}/about"
+                },
+                {
+                    "rel": "self",
+                    "type": "application/activity+json",
+                    "href": f"https://{current_app.config['SERVER_NAME']}/actor",
+                }
+              ]
+            }
+            resp = jsonify(webfinger_data)
+            resp.headers.add_header('Access-Control-Allow-Origin', '*')
+            return resp
+
         seperator = 'u'
         type = 'Person'
         user = User.query.filter_by(user_name=actor.strip(), deleted=False, banned=False, ap_id=None).first()
@@ -1207,8 +1229,12 @@ def user_inbox(actor):
     if site.log_activitypub_json:
         activity_log.activity_json = json.dumps(request_json)
 
-    actor = find_actor_or_create(request_json['actor']) if 'actor' in request_json else None
+    actor = find_actor_or_create(request_json['actor'], signed_get=True) if 'actor' in request_json else None
     if actor is not None:
+        if (('type' in request_json and request_json['type'] == 'Like') or
+                ('type' in request_json and request_json['type'] == 'Undo' and
+                'object' in request_json and request_json['object']['type'] == 'Like')):
+                return shared_inbox()
         try:
             HttpSignature.verify_request(request, actor.public_key, skip_date=True)
             if 'type' in request_json and request_json['type'] == 'Follow':
@@ -1224,10 +1250,6 @@ def user_inbox(actor):
                 else:
                     process_user_undo_follow_request.delay(request_json, activity_log.id, actor.id)
                 return ''
-            if (('type' in request_json and request_json['type'] == 'Like') or
-                ('type' in request_json and request_json['type'] == 'Undo' and
-                'object' in request_json and request_json['object']['type'] == 'Like')):
-                return shared_inbox()
         except VerificationError:
             activity_log.result = 'failure'
             activity_log.exception_message = 'Could not verify signature'
@@ -1257,7 +1279,8 @@ def process_user_follow_request(request_json, activitypublog_id, remote_user_id)
         if not existing_follower:
             auto_accept = not local_user.ap_manually_approves_followers
             new_follower = UserFollower(local_user_id=local_user.id, remote_user_id=remote_user.id, is_accepted=auto_accept)
-            local_user.ap_followers_url = local_user.ap_public_url + '/followers'
+            if not local_user.ap_followers_url:
+                local_user.ap_followers_url = local_user.ap_public_url + '/followers'
             db.session.add(new_follower)
         accept = {
             "@context": default_context(),

app/activitypub/signature.py

@@ -45,7 +45,6 @@ from dateutil import parser
 from pyld import jsonld
 from email.utils import formatdate
 from app import db
-from app.activitypub.util import default_context
 from app.constants import DATETIME_MS_FORMAT
 from app.models import utcnow, ActivityPubLog
 
@@ -106,6 +105,16 @@ def post_request(uri: str, body: dict | None, private_key: str, key_id: str, con
     return log.result != 'failure'
 
 
+def signed_get_request(uri: str, private_key: str, key_id: str, content_type: str = "application/activity+json",
+        method: Literal["get", "post"] = "get", timeout: int = 5,):
+    try:
+        result = HttpSignature.signed_request(uri, None, private_key, key_id, content_type, method, timeout)
+    except Exception as e:
+        current_app.logger.error(f'Exception while sending post to {uri}')
+
+    return result
+
+
 class VerificationError(BaseException):
     """
     There was an error with verifying the signature
@@ -465,3 +474,33 @@ class LDSignature:
         digest = hashes.Hash(hashes.SHA256())
         digest.update(norm_form.encode("utf8"))
         return digest.finalize().hex().encode("ascii")
+
+
+def default_context():
+    context = [
+        "https://www.w3.org/ns/activitystreams",
+        "https://w3id.org/security/v1",
+    ]
+    if current_app.config['FULL_AP_CONTEXT']:
+        context.append({
+            "lemmy": "https://join-lemmy.org/ns#",
+            "litepub": "http://litepub.social/ns#",
+            "pt": "https://joinpeertube.org/ns#",
+            "sc": "http://schema.org/",
+            "ChatMessage": "litepub:ChatMessage",
+            "commentsEnabled": "pt:commentsEnabled",
+            "sensitive": "as:sensitive",
+            "matrixUserId": "lemmy:matrixUserId",
+            "postingRestrictedToMods": "lemmy:postingRestrictedToMods",
+            "removeData": "lemmy:removeData",
+            "stickied": "lemmy:stickied",
+            "moderators": {
+                "@type": "@id",
+                "@id": "lemmy:moderators"
+            },
+            "expires": "as:endTime",
+            "distinguished": "lemmy:distinguished",
+            "language": "sc:inLanguage",
+            "identifier": "sc:identifier"
+        })
+    return context

app/activitypub/util.py

@@ -14,6 +14,7 @@ from app import db, cache, constants, celery
 from app.models import User, Post, Community, BannedInstances, File, PostReply, AllowedInstances, Instance, utcnow, \
     PostVote, PostReplyVote, ActivityPubLog, Notification, Site, CommunityMember, InstanceRole, Report, Conversation, \
     Language
+from app.activitypub.signature import signed_get_request
 import time
 import base64
 import requests
@@ -245,7 +246,7 @@ def instance_allowed(host: str) -> bool:
     return instance is not None
 
 
-def find_actor_or_create(actor: str, create_if_not_found=True, community_only=False) -> Union[User, Community, None]:
+def find_actor_or_create(actor: str, create_if_not_found=True, community_only=False, signed_get=False) -> Union[User, Community, None]:
     actor_url = actor.strip()
     actor = actor.strip().lower()
     user = None
@@ -295,23 +296,38 @@ def find_actor_or_create(actor: str, create_if_not_found=True, community_only=Fa
     else:   # User does not exist in the DB, it's going to need to be created from it's remote home instance
         if create_if_not_found:
             if actor.startswith('https://'):
-                try:
-                    actor_data = get_request(actor_url, headers={'Accept': 'application/activity+json'})
-                except requests.exceptions.ReadTimeout:
-                    time.sleep(randint(3, 10))
+                if not signed_get:
                     try:
                         actor_data = get_request(actor_url, headers={'Accept': 'application/activity+json'})
                     except requests.exceptions.ReadTimeout:
+                        time.sleep(randint(3, 10))
+                        try:
+                            actor_data = get_request(actor_url, headers={'Accept': 'application/activity+json'})
+                        except requests.exceptions.ReadTimeout:
+                            return None
+                    except requests.exceptions.ConnectionError:
                         return None
-                except requests.exceptions.ConnectionError:
-                    return None
-                if actor_data.status_code == 200:
-                    actor_json = actor_data.json()
-                    actor_data.close()
-                    actor_model = actor_json_to_model(actor_json, address, server)
-                    if community_only and not isinstance(actor_model, Community):
+                    if actor_data.status_code == 200:
+                        actor_json = actor_data.json()
+                        actor_data.close()
+                        actor_model = actor_json_to_model(actor_json, address, server)
+                        if community_only and not isinstance(actor_model, Community):
+                            return None
+                        return actor_model
+                else:
+                    try:
+                        site = Site.query.get(1)
+                        actor_data = signed_get_request(actor_url, site.private_key,
+                                        f"https://{current_app.config['SERVER_NAME']}/actor#main-key")
+                        if actor_data.status_code == 200:
+                            actor_json = actor_data.json()
+                            actor_data.close()
+                            actor_model = actor_json_to_model(actor_json, address, server)
+                            if community_only and not isinstance(actor_model, Community):
+                                return None
+                            return actor_model
+                    except Exception:
                         return None
-                    return actor_model
             else:
                 # retrieve user details via webfinger, etc
                 try:
@@ -406,6 +422,13 @@ def refresh_user_profile_task(user_id):
                 actor_data = get_request(user.ap_profile_id, headers={'Accept': 'application/activity+json'})
             except requests.exceptions.ReadTimeout:
                 return
+        except:
+            try:
+                site = Site.query.get(1)
+                actor_data = signed_get_request(user.ap_profile_id, site.private_key,
+                                f"https://{current_app.config['SERVER_NAME']}/actor#main-key")
+            except:
+                return
         if actor_data.status_code == 200:
             activity_json = actor_data.json()
             actor_data.close()
@@ -893,36 +916,6 @@ def parse_summary(user_json) -> str:
         return ''
 
 
-def default_context():
-    context = [
-        "https://www.w3.org/ns/activitystreams",
-        "https://w3id.org/security/v1",
-    ]
-    if current_app.config['FULL_AP_CONTEXT']:
-        context.append({
-            "lemmy": "https://join-lemmy.org/ns#",
-            "litepub": "http://litepub.social/ns#",
-            "pt": "https://joinpeertube.org/ns#",
-            "sc": "http://schema.org/",
-            "ChatMessage": "litepub:ChatMessage",
-            "commentsEnabled": "pt:commentsEnabled",
-            "sensitive": "as:sensitive",
-            "matrixUserId": "lemmy:matrixUserId",
-            "postingRestrictedToMods": "lemmy:postingRestrictedToMods",
-            "removeData": "lemmy:removeData",
-            "stickied": "lemmy:stickied",
-            "moderators": {
-                "@type": "@id",
-                "@id": "lemmy:moderators"
-            },
-            "expires": "as:endTime",
-            "distinguished": "lemmy:distinguished",
-            "language": "sc:inLanguage",
-            "identifier": "sc:identifier"
-        })
-    return context
-
-
 def find_reply_parent(in_reply_to: str) -> Tuple[int, int, int]:
     if 'comment' in in_reply_to:
         parent_comment = PostReply.get_by_ap_id(in_reply_to)

app/admin/routes.py

@@ -9,8 +9,8 @@ from sqlalchemy import text, desc, or_
 
 from app import db, celery, cache
 from app.activitypub.routes import process_inbox_request, process_delete_request
-from app.activitypub.signature import post_request
-from app.activitypub.util import default_context, instance_allowed, instance_blocked
+from app.activitypub.signature import post_request, default_context
+from app.activitypub.util import instance_allowed, instance_blocked
 from app.admin.forms import FederationForm, SiteMiscForm, SiteProfileForm, EditCommunityForm, EditUserForm, \
     EditTopicForm, SendNewsletterForm, AddUserForm
 from app.admin.util import unsubscribe_from_everything_then_delete, unsubscribe_from_community, send_newsletter, \

app/admin/util.py

@@ -6,8 +6,7 @@ from sqlalchemy import text, desc
 from flask_babel import _
 
 from app import db, cache, celery
-from app.activitypub.signature import post_request
-from app.activitypub.util import default_context
+from app.activitypub.signature import post_request, default_context
 from app.models import User, Community, Instance, Site, ActivityPubLog, CommunityMember, Topic
 from app.utils import gibberish, topic_tree
 

app/community/routes.py

@@ -9,8 +9,8 @@ from slugify import slugify
 from sqlalchemy import or_, desc, text
 
 from app import db, constants, cache
-from app.activitypub.signature import RsaKeys, post_request
-from app.activitypub.util import default_context, notify_about_post, make_image_sizes
+from app.activitypub.signature import RsaKeys, post_request, default_context
+from app.activitypub.util import notify_about_post, make_image_sizes
 from app.chat.util import send_message
 from app.community.forms import SearchRemoteCommunity, CreateDiscussionForm, CreateImageForm, CreateLinkForm, \
     ReportCommunityForm, \

app/community/util.py

@@ -9,8 +9,8 @@ from flask_login import current_user
 from pillow_heif import register_heif_opener
 
 from app import db, cache, celery
-from app.activitypub.signature import post_request
-from app.activitypub.util import find_actor_or_create, actor_json_to_model, post_json_to_model, default_context, ensure_domains_match
+from app.activitypub.signature import post_request, default_context
+from app.activitypub.util import find_actor_or_create, actor_json_to_model, post_json_to_model, ensure_domains_match
 from app.constants import POST_TYPE_ARTICLE, POST_TYPE_LINK, POST_TYPE_IMAGE, POST_TYPE_VIDEO, NOTIF_POST
 from app.models import Community, File, BannedInstances, PostReply, PostVote, Post, utcnow, CommunityMember, Site, \
     Instance, Notification, User, ActivityPubLog, NotificationSubscription

app/main/routes.py

@@ -9,8 +9,9 @@ import requests
 from sqlalchemy.sql.operators import or_, and_
 
 from app import db, cache
-from app.activitypub.util import default_context, make_image_sizes_async, refresh_user_profile, find_actor_or_create, \
+from app.activitypub.util import make_image_sizes_async, refresh_user_profile, find_actor_or_create, \
     refresh_community_profile_task, users_total, active_month, local_posts, local_communities, local_comments
+from app.activitypub.signature import default_context
 from app.constants import SUBSCRIPTION_PENDING, SUBSCRIPTION_MEMBER, POST_TYPE_IMAGE, POST_TYPE_LINK, \
     SUBSCRIPTION_OWNER, SUBSCRIPTION_MODERATOR, POST_TYPE_VIDEO
 from app.email import send_email, send_welcome_email
@@ -414,3 +415,30 @@ def activitypub_application():
     resp = jsonify(application_data)
     resp.content_type = 'application/activity+json'
     return resp
+
+
+# instance actor (literally uses the word 'actor' without the /u/)
+# required for interacting with instances using 'secure mode' (aka authorized fetch)
+@bp.route('/actor', methods=['GET'])
+def instance_actor():
+    application_data = {
+        '@context': default_context(),
+        'type': 'Application',
+        'id': f"https://{current_app.config['SERVER_NAME']}/actor",
+        'preferredUsername': f"{current_app.config['SERVER_NAME']}",
+        'url': f"https://{current_app.config['SERVER_NAME']}/about",
+        'manuallyApprovesFollowers': True,
+        'inbox': f"https://{current_app.config['SERVER_NAME']}/actor/inbox",
+        'outbox': f"https://{current_app.config['SERVER_NAME']}/actor/outbox",
+        'publicKey': {
+          'id': f"https://{current_app.config['SERVER_NAME']}/actor#main-key",
+          'owner': f"https://{current_app.config['SERVER_NAME']}/actor",
+          'publicKeyPem': g.site.public_key
+        },
+        'endpoints': {
+          'sharedInbox': f"https://{current_app.config['SERVER_NAME']}/site_inbox",
+        }
+    }
+    resp = jsonify(application_data)
+    resp.content_type = 'application/activity+json'
+    return resp

app/post/routes.py

@@ -8,8 +8,8 @@ from flask_babel import _
 from sqlalchemy import or_, desc
 
 from app import db, constants, cache
-from app.activitypub.signature import HttpSignature, post_request
-from app.activitypub.util import default_context, notify_about_post_reply
+from app.activitypub.signature import HttpSignature, post_request, default_context
+from app.activitypub.util import notify_about_post_reply
 from app.community.util import save_post, send_to_remote_instance
 from app.inoculation import inoculation
 from app.post.forms import NewReplyForm, ReportPostForm, MeaCulpaForm

app/user/routes.py

@@ -6,8 +6,8 @@ from flask_login import login_user, logout_user, current_user, login_required
 from flask_babel import _
 
 from app import db, cache, celery
-from app.activitypub.signature import post_request
-from app.activitypub.util import default_context, find_actor_or_create
+from app.activitypub.signature import post_request, default_context
+from app.activitypub.util import find_actor_or_create
 from app.community.util import save_icon_file, save_banner_file, retrieve_mods_and_backfill
 from app.constants import SUBSCRIPTION_MEMBER, SUBSCRIPTION_PENDING, NOTIF_USER
 from app.models import Post, Community, CommunityMember, User, PostReply, PostVote, Notification, utcnow, File, Site, \

app/user/utils.py

@@ -3,8 +3,8 @@ from time import sleep
 from flask import current_app, json
 
 from app import celery, db
-from app.activitypub.signature import post_request
-from app.activitypub.util import default_context, actor_json_to_model
+from app.activitypub.signature import post_request, default_context
+from app.activitypub.util import actor_json_to_model
 from app.community.util import send_to_remote_instance
 from app.models import User, CommunityMember, Community, Instance, Site, utcnow, ActivityPubLog, BannedInstances
 from app.utils import gibberish, ap_datetime, instance_banned, get_request