michael/pyfedi
1
0
Fork 0
mirror of https://codeberg.org/rimu/pyfedi synced 2025-01-23 11:26:56 -08:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'origin/main'

Browse source
This commit is contained in:
rimu 2025-01-21 19:51:22 +13:00
commit b98e866a14
17 changed files with 964 additions and 497 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
app/activitypub/routes.py
View file

@ -415,16 +415,19 @@ def shared_inbox():
return '', 200
id = request_json['id']
missing_actor_in_announce_object = False # nodebb
if request_json['type'] == 'Announce' and isinstance(request_json['object'], dict):
object = request_json['object']
if not 'id' in object or not 'type' in object or not 'actor' in object or not 'object' in object:
if not 'actor' in object:
missing_actor_in_announce_object = True
if not 'id' in object or not 'type' in object or not 'object' in object:
if 'type' in object and (object['type'] == 'Page' or object['type'] == 'Note'):
log_incoming_ap(id, APLOG_ANNOUNCE, APLOG_IGNORED, saved_json, 'Intended for Mastodon')
else:
log_incoming_ap(id, APLOG_ANNOUNCE, APLOG_FAILURE, saved_json, 'Missing minimum expected fields in JSON Announce object')
return '', 200
if isinstance(object['actor'], str) and object['actor'].startswith('https://' + current_app.config['SERVER_NAME']):
if not missing_actor_in_announce_object and isinstance(object['actor'], str) and object['actor'].startswith('https://' + current_app.config['SERVER_NAME']):
log_incoming_ap(id, APLOG_DUPLICATE, APLOG_IGNORED, saved_json, 'Activity about local content which is already present')
return '', 200
@ -499,6 +502,11 @@ def shared_inbox():
process_delete_request.delay(request_json, store_ap_json)
return ''
if missing_actor_in_announce_object:
if ((request_json['object']['type'] == 'Create' or request_json['object']['type']) and
'attributedTo' in request_json['object']['object'] and isinstance(request_json['object']['object']['attributedTo'], str)):
request_json['object']['actor'] = request_json['object']['object']['attributedTo']
if current_app.debug:
process_inbox_request(request_json, store_ap_json)
else:
@ -528,17 +536,20 @@ def replay_inbox_request(request_json):
return
id = request_json['id']
missing_actor_in_announce_object = False # nodebb
if request_json['type'] == 'Announce' and isinstance(request_json['object'], dict):
object = request_json['object']
if not 'id' in object or not 'type' in object or not 'actor' in object or not 'object' in object:
if not 'actor' in object:
missing_actor_in_announce_object = True
if not 'id' in object or not 'type' in object or not 'object' in object:
if 'type' in object and (object['type'] == 'Page' or object['type'] == 'Note'):
log_incoming_ap(id, APLOG_ANNOUNCE, APLOG_IGNORED, request_json, 'REPLAY: Intended for Mastodon')
else:
log_incoming_ap(id, APLOG_ANNOUNCE, APLOG_FAILURE, request_json, 'REPLAY: Missing minimum expected fields in JSON Announce object')
return
if isinstance(object['actor'], str) and object['actor'].startswith('https://' + current_app.config['SERVER_NAME']):
log_incoming_ap(id, APLOG_DUPLICATE, APLOG_IGNORED, saved_json, 'Activity about local content which is already present')
if not missing_actor_in_announce_object and isinstance(object['actor'], str) and object['actor'].startswith('https://' + current_app.config['SERVER_NAME']):
log_incoming_ap(id, APLOG_DUPLICATE, APLOG_IGNORED, request_json, 'REPLAY: Activity about local content which is already present')
return
# Ignore unutilised PeerTube activity
@ -573,6 +584,11 @@ def replay_inbox_request(request_json):
process_delete_request(request_json, True)
return
if missing_actor_in_announce_object:
if ((request_json['object']['type'] == 'Create' or request_json['object']['type']) and
'attributedTo' in request_json['object']['object'] and isinstance(request_json['object']['object']['attributedTo'], str)):
request_json['object']['actor'] = request_json['object']['object']['attributedTo']
process_inbox_request(request_json, True)
return
@ -614,7 +630,7 @@ def process_inbox_request(request_json, store_ap_json):
if request_json['object'].startswith('https://' + current_app.config['SERVER_NAME']):
log_incoming_ap(id, APLOG_DUPLICATE, APLOG_IGNORED, saved_json, 'Activity about local content which is already present')
return
post = resolve_remote_post(request_json['object'], community.id, announce_actor=community.ap_profile_id, store_ap_json=store_ap_json)
post = resolve_remote_post(request_json['object'], community, id, store_ap_json)
if post:
log_incoming_ap(id, APLOG_ANNOUNCE, APLOG_SUCCESS, request_json)
else:
@ -799,7 +815,8 @@ def process_inbox_request(request_json, store_ap_json):
# inner object of Create is not a ChatMessage
else:
if (core_activity['object']['type'] == 'Note' and 'name' in core_activity['object'] and # Poll Votes
'inReplyTo' in core_activity['object'] and 'attributedTo' in core_activity['object']):
'inReplyTo' in core_activity['object'] and 'attributedTo' in core_activity['object'] and
not 'published' in core_activity['object']):
post_being_replied_to = Post.query.filter_by(ap_id=core_activity['object']['inReplyTo']).first()
if post_being_replied_to:
poll_data = Poll.query.get(post_being_replied_to.id)

392
app/activitypub/util.py
View file

@ -521,7 +521,7 @@ def refresh_user_profile_task(user_id):
if field_data['type'] == 'PropertyValue':
if '<a ' in field_data['value']:
field_data['value'] = mastodon_extra_field_link(field_data['value'])
user.extra_fields.append(UserExtraField(label=field_data['name'].strip(), text=field_data['value'].strip()))
user.extra_fields.append(UserExtraField(shorten_string(label=field_data['name'].strip()), text=field_data['value'].strip()))
if 'type' in activity_json:
user.bot = True if activity_json['type'] == 'Service' else False
user.ap_fetched_at = utcnow()
@ -772,7 +772,7 @@ def actor_json_to_model(activity_json, address, server):
if field_data['type'] == 'PropertyValue':
if '<a ' in field_data['value']:
field_data['value'] = mastodon_extra_field_link(field_data['value'])
user.extra_fields.append(UserExtraField(label=field_data['name'].strip(), text=field_data['value'].strip()))
user.extra_fields.append(UserExtraField(label=shorten_string(field_data['name'].strip()), text=field_data['value'].strip()))
try:
db.session.add(user)
db.session.commit()
@ -885,120 +885,6 @@ def actor_json_to_model(activity_json, address, server):
return community
def post_json_to_model(activity_log, post_json, user, community) -> Post:
try:
nsfl_in_title = '[NSFL]' in post_json['name'].upper() or '(NSFL)' in post_json['name'].upper()
post = Post(user_id=user.id, community_id=community.id,
title=html.unescape(post_json['name']),
comments_enabled=post_json['commentsEnabled'] if 'commentsEnabled' in post_json else True,
sticky=post_json['stickied'] if 'stickied' in post_json else False,
nsfw=post_json['sensitive'],
nsfl=post_json['nsfl'] if 'nsfl' in post_json else nsfl_in_title,
ap_id=post_json['id'],
type=constants.POST_TYPE_ARTICLE,
posted_at=post_json['published'],
last_active=post_json['published'],
instance_id=user.instance_id,
indexable = user.indexable
)
if 'content' in post_json:
if post_json['mediaType'] == 'text/html':
post.body_html = allowlist_html(post_json['content'])
if 'source' in post_json and post_json['source']['mediaType'] == 'text/markdown':
post.body = post_json['source']['content']
post.body_html = markdown_to_html(post.body) # prefer Markdown if provided, overwrite version obtained from HTML
else:
post.body = html_to_text(post.body_html)
elif post_json['mediaType'] == 'text/markdown':
post.body = post_json['content']
post.body_html = markdown_to_html(post.body)
if 'attachment' in post_json and len(post_json['attachment']) > 0 and 'type' in post_json['attachment'][0]:
alt_text = None
if post_json['attachment'][0]['type'] == 'Link':
post.url = post_json['attachment'][0]['href'] # Lemmy < 0.19.4
if post_json['attachment'][0]['type'] == 'Image':
post.url = post_json['attachment'][0]['url'] # PieFed, Lemmy >= 0.19.4
if 'name' in post_json['attachment'][0]:
alt_text = post_json['attachment'][0]['name']
if post.url:
if is_image_url(post.url):
post.type = POST_TYPE_IMAGE
image = File(source_url=post.url)
if alt_text:
image.alt_text = alt_text
db.session.add(image)
post.image = image
elif is_video_url(post.url):
post.type = POST_TYPE_VIDEO
else:
post.type = POST_TYPE_LINK
post.url = remove_tracking_from_link(post.url)
domain = domain_from_url(post.url)
# notify about links to banned websites.
already_notified = set() # often admins and mods are the same people - avoid notifying them twice
if domain:
if domain.notify_mods:
for community_member in post.community.moderators():
notify = Notification(title='Suspicious content', url=post.ap_id, user_id=community_member.user_id, author_id=user.id)
db.session.add(notify)
already_notified.add(community_member.user_id)
if domain.notify_admins:
for admin in Site.admins():
if admin.id not in already_notified:
notify = Notification(title='Suspicious content', url=post.ap_id, user_id=admin.id, author_id=user.id)
db.session.add(notify)
admin.unread_notifications += 1
if domain.banned:
post = None
activity_log.exception_message = domain.name + ' is blocked by admin'
if not domain.banned:
domain.post_count += 1
post.domain = domain
if post is not None:
if post_json['type'] == 'Video':
post.type = POST_TYPE_VIDEO
post.url = post_json['id']
if 'icon' in post_json and isinstance(post_json['icon'], list):
icon = File(source_url=post_json['icon'][-1]['url'])
db.session.add(icon)
post.image = icon
if 'language' in post_json:
language = find_language_or_create(post_json['language']['identifier'], post_json['language']['name'])
if language:
post.language_id = language.id
if 'tag' in post_json:
for json_tag in post_json['tag']:
if json_tag['type'] == 'Hashtag':
# Lemmy adds the community slug as a hashtag on every post in the community, which we want to ignore
if json_tag['name'][1:].lower() != community.name.lower():
hashtag = find_hashtag_or_create(json_tag['name'])
if hashtag:
post.tags.append(hashtag)
if 'image' in post_json and post.image is None:
image = File(source_url=post_json['image']['url'])
db.session.add(image)
post.image = image
db.session.add(post)
community.post_count += 1
user.post_count += 1
activity_log.result = 'success'
db.session.commit()
if post.image_id:
make_image_sizes(post.image_id, 170, 512, 'posts') # the 512 sized image is for masonry view
return post
except KeyError as e:
current_app.logger.error(f'KeyError in post_json_to_model: ' + str(post_json))
return None
# Save two different versions of a File, after downloading it from file.source_url. Set a width parameter to None to avoid generating one of that size
def make_image_sizes(file_id, thumbnail_width=50, medium_width=120, directory='posts', toxic_community=False):
if current_app.debug:
@ -2295,125 +2181,7 @@ def can_delete(user_ap_id, post):
return can_edit(user_ap_id, post)
def resolve_remote_post(uri: str, community_id: int, announce_actor=None, store_ap_json=False) -> Union[Post, PostReply, None]:
post = Post.query.filter_by(ap_id=uri).first()
if post:
return post
community = Community.query.get(community_id)
site = Site.query.get(1)
parsed_url = urlparse(uri)
uri_domain = parsed_url.netloc
if announce_actor:
parsed_url = urlparse(announce_actor)
announce_actor_domain = parsed_url.netloc
if announce_actor_domain != 'a.gup.pe' and announce_actor_domain != uri_domain:
return None
actor_domain = None
actor = None
post_request = get_request(uri, headers={'Accept': 'application/activity+json'})
if post_request.status_code == 200:
post_data = post_request.json()
post_request.close()
# check again that it doesn't already exist (can happen with different but equivalent URLs)
post = Post.query.filter_by(ap_id=post_data['id']).first()
if post:
return post
if 'attributedTo' in post_data:
if isinstance(post_data['attributedTo'], str):
actor = post_data['attributedTo']
parsed_url = urlparse(post_data['attributedTo'])
actor_domain = parsed_url.netloc
elif isinstance(post_data['attributedTo'], list):
for a in post_data['attributedTo']:
if a['type'] == 'Person':
actor = a['id']
parsed_url = urlparse(a['id'])
actor_domain = parsed_url.netloc
break
if uri_domain != actor_domain:
return None
if not announce_actor:
# make sure that the post actually belongs in the community a user says it does
remote_community = None
if post_data['type'] == 'Page': # lemmy
remote_community = post_data['audience'] if 'audience' in post_data else None
if remote_community and remote_community.lower() != community.ap_profile_id:
return None
elif post_data['type'] == 'Video': # peertube
if 'attributedTo' in post_data and isinstance(post_data['attributedTo'], list):
for a in post_data['attributedTo']:
if a['type'] == 'Group':
remote_community = a['id']
break
if remote_community and remote_community.lower() != community.ap_profile_id:
return None
else: # mastodon, etc
if 'inReplyTo' not in post_data or post_data['inReplyTo'] != None:
return None
community_found = False
if not community_found and 'to' in post_data and isinstance(post_data['to'], str):
remote_community = post_data['to']
if remote_community.lower() == community.ap_profile_id:
community_found = True
if not community_found and 'cc' in post_data and isinstance(post_data['cc'], str):
remote_community = post_data['cc']
if remote_community.lower() == community.ap_profile_id:
community_found = True
if not community_found and 'to' in post_data and isinstance(post_data['to'], list):
for t in post_data['to']:
if t.lower() == community.ap_profile_id:
community_found = True
break
if not community_found and 'cc' in post_data and isinstance(post_data['cc'], list):
for c in post_data['cc']:
if c.lower() == community.ap_profile_id:
community_found = True
break
if not community_found:
return None
user = find_actor_or_create(actor)
if user and community and post_data:
request_json = {
'id': f"https://{uri_domain}/activities/create/{gibberish(15)}",
'object': post_data
}
if 'inReplyTo' in request_json['object'] and request_json['object']['inReplyTo']:
post_reply = create_post_reply(store_ap_json, community, request_json['object']['inReplyTo'], request_json, user)
if post_reply:
if 'published' in post_data:
post_reply.posted_at = post_data['published']
post_reply.post.last_active = post_data['published']
post_reply.community.last_active = utcnow()
db.session.commit()
return post_reply
else:
post = create_post(store_ap_json, community, request_json, user)
if post:
if 'published' in post_data:
post.posted_at=post_data['published']
post.last_active=post_data['published']
post.community.last_active = utcnow()
db.session.commit()
return post
return None
# called from UI, via 'search' option in navbar
def resolve_remote_post_from_search(uri: str) -> Union[Post, None]:
post = Post.query.filter_by(ap_id=uri).first()
if post:
return post
parsed_url = urlparse(uri)
uri_domain = parsed_url.netloc
actor_domain = None
actor = None
def remote_object_to_json(uri):
try:
object_request = get_request(uri, headers={'Accept': 'application/activity+json'})
except httpx.HTTPError:
@ -2424,7 +2192,8 @@ def resolve_remote_post_from_search(uri: str) -> Union[Post, None]:
return None
if object_request.status_code == 200:
try:
post_data = object_request.json()
object = object_request.json()
return object
except:
object_request.close()
return None
@ -2440,7 +2209,8 @@ def resolve_remote_post_from_search(uri: str) -> Union[Post, None]:
except httpx.HTTPError:
return None
try:
post_data = object_request.json()
object = object_request.json()
return object
except:
object_request.close()
return None
@ -2448,8 +2218,129 @@ def resolve_remote_post_from_search(uri: str) -> Union[Post, None]:
else:
return None
# called from incoming activitypub, when the object in an Announce is just a URL
# despite the name, it works for both posts and replies
def resolve_remote_post(uri: str, community, announce_id, store_ap_json, nodebb=False) -> Union[Post, PostReply, None]:
parsed_url = urlparse(uri)
uri_domain = parsed_url.netloc
announce_actor = community.ap_profile_id
parsed_url = urlparse(announce_actor)
announce_actor_domain = parsed_url.netloc
if announce_actor_domain != 'a.gup.pe' and not nodebb and announce_actor_domain != uri_domain:
return None
actor_domain = None
actor = None
post_data = remote_object_to_json(uri)
# find the author. Make sure their domain matches the site hosting it to mitigate impersonation attempts
if 'attributedTo' in post_data:
attributed_to = post_data['attributedTo']
if isinstance(attributed_to, str):
actor = attributed_to
parsed_url = urlparse(actor)
actor_domain = parsed_url.netloc
elif isinstance(attributed_to, list):
for a in attributed_to:
if isinstance(a, dict) and a.get('type') == 'Person':
actor = a.get('id')
if isinstance(actor, str): # Ensure `actor` is a valid string
parsed_url = urlparse(actor)
actor_domain = parsed_url.netloc
break
elif isinstance(a, str):
actor = a
parsed_url = urlparse(actor)
actor_domain = parsed_url.netloc
break
if uri_domain != actor_domain:
return None
user = find_actor_or_create(actor)
if user and community and post_data:
activity = 'update' if 'updated' in post_data else 'create'
request_json = {'id': f"https://{uri_domain}/activities/{activity}/{gibberish(15)}", 'object': post_data}
if 'inReplyTo' in request_json['object'] and request_json['object']['inReplyTo']:
if activity == 'update':
post_reply = PostReply.get_by_ap_id(uri)
if post_reply:
update_post_reply_from_activity(post_reply, request_json)
else:
activity = 'create'
if activity == 'create':
post_reply = create_post_reply(store_ap_json, community, request_json['object']['inReplyTo'], request_json, user)
if post_reply:
if 'published' in post_data:
post_reply.posted_at = post_data['published']
post_reply.post.last_active = post_data['published']
post_reply.community.last_active = utcnow()
db.session.commit()
if post_reply:
return post_reply
else:
if activity == 'update':
post = Post.get_by_ap_id(uri)
if post:
update_post_from_activity(post, request_json)
else:
activity = 'create'
if activity == 'create':
post = create_post(store_ap_json, community, request_json, user, announce_id)
if post:
if 'published' in post_data:
post.posted_at=post_data['published']
post.last_active=post_data['published']
post.community.last_active = utcnow()
db.session.commit()
if post:
return post
return None
@celery.task
def get_nodebb_replies_in_background(replies_uri_list, community_id):
max = 10 if not current_app.debug else 2 # magic number alert
community = Community.query.get(community_id)
if not community:
return
reply_count = 0
for uri in replies_uri_list:
reply_count += 1
post_reply = resolve_remote_post(uri, community, None, False, nodebb=True)
if reply_count >= max:
break
# called from UI, via 'search' option in navbar, or 'Retrieve a post from the original server' in community sidebar
def resolve_remote_post_from_search(uri: str) -> Union[Post, None]:
post = Post.get_by_ap_id(uri)
if post:
return post
parsed_url = urlparse(uri)
uri_domain = parsed_url.netloc
actor_domain = None
actor = None
post_data = remote_object_to_json(uri)
# nodebb. the post is the first entry in orderedItems of a topic, and the replies are the remaining entries
# just gets orderedItems[0] to retrieve the post, and then replies are retrieved in the background
topic_post_data = post_data
nodebb = False
if ('type' in post_data and post_data['type'] == 'OrderedCollection' and
'totalItems' in post_data and post_data['totalItems'] > 0 and
'orderedItems' in post_data and isinstance(post_data['orderedItems'], list)):
nodebb = True
uri = post_data['orderedItems'][0]
parsed_url = urlparse(uri)
uri_domain = parsed_url.netloc
post_data = remote_object_to_json(uri)
# check again that it doesn't already exist (can happen with different but equivalent URLs)
post = Post.query.filter_by(ap_id=post_data['id']).first()
post = Post.get_by_ap_id(post_data['id'])
if post:
return post
@ -2478,16 +2369,23 @@ def resolve_remote_post_from_search(uri: str) -> Union[Post, None]:
# find the community the post was submitted to
community = find_community(post_data)
if not community and nodebb:
community = find_community(topic_post_data) # use 'audience' from topic if post has no info for how it got there
# find the post's author
user = find_actor_or_create(actor)
if user and community and post_data:
request_json = {'id': f"https://{uri_domain}/activities/create/gibberish(15)", 'object': post_data}
request_json = {'id': f"https://{uri_domain}/activities/create/{gibberish(15)}", 'object': post_data}
post = create_post(False, community, request_json, user)
if post:
if 'published' in post_data:
post.posted_at=post_data['published']
post.last_active=post_data['published']
db.session.commit()
if nodebb and topic_post_data['totalItems'] > 1:
if current_app.debug:
get_nodebb_replies_in_background(topic_post_data['orderedItems'][1:], community.id)
else:
get_nodebb_replies_in_background.delay(topic_post_data['orderedItems'][1:], community.id)
return post
return None
@ -2622,6 +2520,8 @@ def log_incoming_ap(id, aplog_type, aplog_result, saved_json, message=None):
def find_community(request_json):
# Create/Update from platform that included Community in 'audience', 'cc', or 'to' in outer or inner object
# Also works for manually retrieved posts
locations = ['audience', 'cc', 'to']
if 'object' in request_json and isinstance(request_json['object'], dict):
rjs = [request_json, request_json['object']]
@ -2643,9 +2543,20 @@ def find_community(request_json):
if potential_community:
return potential_community
# used for manual retrieval of a PeerTube vid
if request_json['type'] == 'Video':
if 'attributedTo' in request_json and isinstance(request_json['attributedTo'], list):
for a in request_json['attributedTo']:
if a['type'] == 'Group':
potential_community = Community.query.filter_by(ap_profile_id=a['id'].lower()).first()
if potential_community:
return potential_community
# change this if manual retrieval of comments is allowed in future
if not 'object' in request_json:
return None
# Create/Update Note from platform that didn't include the Community in 'audience', 'cc', or 'to' (e.g. Mastodon reply to Lemmy post)
if 'inReplyTo' in request_json['object'] and request_json['object']['inReplyTo'] is not None:
post_being_replied_to = Post.query.filter_by(ap_id=request_json['object']['inReplyTo'].lower()).first()
if post_being_replied_to:
@ -2655,7 +2566,8 @@ def find_community(request_json):
if comment_being_replied_to:
return comment_being_replied_to.community
if request_json['object']['type'] == 'Video': # PeerTube
# Update / Video from PeerTube (possibly an edit, more likely an invite to query Likes / Replies endpoints)
if request_json['object']['type'] == 'Video':
if 'attributedTo' in request_json['object'] and isinstance(request_json['object']['attributedTo'], list):
for a in request_json['object']['attributedTo']:
if a['type'] == 'Group':

189
app/api/alpha/routes.py
View file

@ -1,8 +1,10 @@
from app.api.alpha import bp
from app.api.alpha.utils import get_site, post_site_block, \
get_search, \
get_post_list, get_post, post_post_like, put_post_save, put_post_subscribe, post_post, put_post, post_post_delete, post_post_report, \
get_reply_list, post_reply_like, put_reply_save, put_reply_subscribe, post_reply, put_reply, post_reply_delete, post_reply_report, \
get_post_list, get_post, post_post_like, put_post_save, put_post_subscribe, post_post, \
put_post, post_post_delete, post_post_report, post_post_lock, post_post_feature, post_post_remove, \
get_reply_list, post_reply_like, put_reply_save, put_reply_subscribe, post_reply, put_reply, \
post_reply_delete, post_reply_report, post_reply_remove, \
get_community_list, get_community, post_community_follow, post_community_block, \
get_user, post_user_block
from app.shared.auth import log_user_in
@ -207,6 +209,42 @@ def post_alpha_post_report():
return jsonify({"error": str(ex)}), 400
@bp.route('/api/alpha/post/lock', methods=['POST'])
def post_alpha_post_lock():
if not enable_api():
return jsonify({'error': 'alpha api is not enabled'})
try:
auth = request.headers.get('Authorization')
data = request.get_json(force=True) or {}
return jsonify(post_post_lock(auth, data))
except Exception as ex:
return jsonify({"error": str(ex)}), 400
@bp.route('/api/alpha/post/feature', methods=['POST'])
def post_alpha_post_feature():
if not enable_api():
return jsonify({'error': 'alpha api is not enabled'})
try:
auth = request.headers.get('Authorization')
data = request.get_json(force=True) or {}
return jsonify(post_post_feature(auth, data))
except Exception as ex:
return jsonify({"error": str(ex)}), 400
@bp.route('/api/alpha/post/remove', methods=['POST'])
def post_alpha_post_remove():
if not enable_api():
return jsonify({'error': 'alpha api is not enabled'})
try:
auth = request.headers.get('Authorization')
data = request.get_json(force=True) or {}
return jsonify(post_post_remove(auth, data))
except Exception as ex:
return jsonify({"error": str(ex)}), 400
# Reply
@bp.route('/api/alpha/comment/list', methods=['GET'])
def get_alpha_comment_list():
@ -304,6 +342,18 @@ def post_alpha_comment_report():
return jsonify({"error": str(ex)}), 400
@bp.route('/api/alpha/comment/remove', methods=['POST'])
def post_alpha_comment_remove():
if not enable_api():
return jsonify({'error': 'alpha api is not enabled'})
try:
auth = request.headers.get('Authorization')
data = request.get_json(force=True) or {}
return jsonify(post_reply_remove(auth, data))
except Exception as ex:
return jsonify({"error": str(ex)}), 400
# User
@bp.route('/api/alpha/user', methods=['GET'])
def get_alpha_user():
@ -344,109 +394,104 @@ def post_alpha_user_block():
# Not yet implemented. Copied from lemmy's V3 api, so some aren't needed, and some need changing
# Site - not yet implemented
@bp.route('/api/alpha/site', methods=['POST'])
@bp.route('/api/alpha/site', methods=['PUT'])
@bp.route('/api/alpha/site', methods=['POST']) # Create New Site. No plans to implement
@bp.route('/api/alpha/site', methods=['PUT']) # Edit Site. Not available in app
def alpha_site():
return jsonify({"error": "not_yet_implemented"}), 400
# Miscellaneous - not yet implemented
@bp.route('/api/alpha/modlog', methods=['GET'])
@bp.route('/api/alpha/resolve_object', methods=['GET'])
@bp.route('/api/alpha/federated_instances', methods=['GET'])
@bp.route('/api/alpha/modlog', methods=['GET']) # Get Modlog. Not usually public
@bp.route('/api/alpha/resolve_object', methods=['GET']) # Stage 1: Needed for search
@bp.route('/api/alpha/federated_instances', methods=['GET']) # No plans to implement - only V3 version needed
def alpha_miscellaneous():
return jsonify({"error": "not_yet_implemented"}), 400
# Community - not yet implemented
@bp.route('/api/alpha/community', methods=['POST'])
@bp.route('/api/alpha/community', methods=['PUT'])
@bp.route('/api/alpha/community/hide', methods=['PUT'])
@bp.route('/api/alpha/community/delete', methods=['POST'])
@bp.route('/api/alpha/community/remove', methods=['POST'])
@bp.route('/api/alpha/community/transfer', methods=['POST'])
@bp.route('/api/alpha/community/ban_user', methods=['POST'])
@bp.route('/api/alpha/community/mod', methods=['POST'])
@bp.route('/api/alpha/community', methods=['POST']) # (none
@bp.route('/api/alpha/community', methods=['PUT']) # of
@bp.route('/api/alpha/community/hide', methods=['PUT']) # these
@bp.route('/api/alpha/community/delete', methods=['POST']) # are
@bp.route('/api/alpha/community/remove', methods=['POST']) # available
@bp.route('/api/alpha/community/transfer', methods=['POST']) # in
@bp.route('/api/alpha/community/ban_user', methods=['POST']) # the
@bp.route('/api/alpha/community/mod', methods=['POST']) # app)
def alpha_community():
return jsonify({"error": "not_yet_implemented"}), 400
# Post - not yet implemented
@bp.route('/api/alpha/post/remove', methods=['POST'])
@bp.route('/api/alpha/post/lock', methods=['POST'])
@bp.route('/api/alpha/post/feature', methods=['POST'])
@bp.route('/api/alpha/post/report', methods=['POST'])
@bp.route('/api/alpha/post/report/resolve', methods=['PUT'])
@bp.route('/api/alpha/post/report/list', methods=['GET'])
@bp.route('/api/alpha/post/site_metadata', methods=['GET'])
@bp.route('/api/alpha/post/report/resolve', methods=['PUT']) # Stage 2
@bp.route('/api/alpha/post/report/list', methods=['GET']) # Stage 2
@bp.route('/api/alpha/post/site_metadata', methods=['GET']) # Not available in app
def alpha_post():
return jsonify({"error": "not_yet_implemented"}), 400
# Reply - not yet implemented
@bp.route('/api/alpha/comment', methods=['GET'])
@bp.route('/api/alpha/comment/remove', methods=['POST'])
@bp.route('/api/alpha/comment/mark_as_read', methods=['POST'])
@bp.route('/api/alpha/comment/distinguish', methods=['POST'])
@bp.route('/api/alpha/comment/report/resolve', methods=['PUT'])
@bp.route('/api/alpha/comment/report/list', methods=['GET'])
@bp.route('/api/alpha/comment', methods=['GET']) # Stage 1 if needed for search
@bp.route('/api/alpha/comment/mark_as_read', methods=['POST']) # No DB support
@bp.route('/api/alpha/comment/distinguish', methods=['POST']) # Not really used
@bp.route('/api/alpha/comment/report/resolve', methods=['PUT']) # Stage 2
@bp.route('/api/alpha/comment/report/list', methods=['GET']) # Stage 2
def alpha_reply():
return jsonify({"error": "not_yet_implemented"}), 400
# Chat - not yet implemented
@bp.route('/api/alpha/private_message/list', methods=['GET'])
@bp.route('/api/alpha/private_message', methods=['PUT'])
@bp.route('/api/alpha/private_message', methods=['POST'])
@bp.route('/api/alpha/private_message/delete', methods=['POST'])
@bp.route('/api/alpha/private_message/mark_as_read', methods=['POST'])
@bp.route('/api/alpha/private_message/report', methods=['POST'])
@bp.route('/api/alpha/private_message/report/resolve', methods=['PUT'])
@bp.route('/api/alpha/private_message/report/list', methods=['GET'])
@bp.route('/api/alpha/private_message/list', methods=['GET']) # Stage 1
@bp.route('/api/alpha/private_message', methods=['PUT']) # Stage 1
@bp.route('/api/alpha/private_message', methods=['POST']) # Stage 1
@bp.route('/api/alpha/private_message/delete', methods=['POST']) # Stage 1
@bp.route('/api/alpha/private_message/mark_as_read', methods=['POST']) # Stage 1
@bp.route('/api/alpha/private_message/report', methods=['POST']) # Stage 1
@bp.route('/api/alpha/private_message/report/resolve', methods=['PUT']) # Stage 2
@bp.route('/api/alpha/private_message/report/list', methods=['GET']) # Stage 2
def alpha_chat():
return jsonify({"error": "not_yet_implemented"}), 400
# User - not yet implemented
@bp.route('/api/alpha/user/register', methods=['POST'])
@bp.route('/api/alpha/user/get_captcha', methods=['GET'])
@bp.route('/api/alpha/user/mention', methods=['GET'])
@bp.route('/api/alpha/user/mention/mark_as_read', methods=['POST'])
@bp.route('/api/alpha/user/replies', methods=['GET'])
@bp.route('/api/alpha/user/ban', methods=['POST'])
@bp.route('/api/alpha/user/banned', methods=['GET'])
@bp.route('/api/alpha/user/delete_account', methods=['POST'])
@bp.route('/api/alpha/user/password_reset', methods=['POST'])
@bp.route('/api/alpha/user/password_change', methods=['POST'])
@bp.route('/api/alpha/user/mark_all_as_read', methods=['POST'])
@bp.route('/api/alpha/user/save_user_settings', methods=['PUT'])
@bp.route('/api/alpha/user/change_password', methods=['PUT'])
@bp.route('/api/alpha/user/repost_count', methods=['GET'])
@bp.route('/api/alpha/user/unread_count', methods=['GET'])
@bp.route('/api/alpha/user/verify_email', methods=['POST'])
@bp.route('/api/alpha/user/leave_admin', methods=['POST'])
@bp.route('/api/alpha/user/totp/generate', methods=['POST'])
@bp.route('/api/alpha/user/totp/update', methods=['POST'])
@bp.route('/api/alpha/user/export_settings', methods=['GET'])
@bp.route('/api/alpha/user/import_settings', methods=['POST'])
@bp.route('/api/alpha/user/list_logins', methods=['GET'])
@bp.route('/api/alpha/user/validate_auth', methods=['GET'])
@bp.route('/api/alpha/user/logout', methods=['POST'])
@bp.route('/api/alpha/user/register', methods=['POST']) # Not available in app
@bp.route('/api/alpha/user/get_captcha', methods=['GET']) # Not available in app
@bp.route('/api/alpha/user/mention', methods=['GET']) # No DB support
@bp.route('/api/alpha/user/mention/mark_as_read', methods=['POST']) # No DB support
@bp.route('/api/alpha/user/replies', methods=['GET']) # Stage 1
@bp.route('/api/alpha/user/ban', methods=['POST']) # Admin function. No plans to implement
@bp.route('/api/alpha/user/banned', methods=['GET']) # Admin function. No plans to implement
@bp.route('/api/alpha/user/delete_account', methods=['POST']) # Not available in app
@bp.route('/api/alpha/user/password_reset', methods=['POST']) # Not available in app
@bp.route('/api/alpha/user/password_change', methods=['POST']) # Not available in app
@bp.route('/api/alpha/user/mark_all_as_read', methods=['POST']) # Stage 1
@bp.route('/api/alpha/user/save_user_settings', methods=['PUT']) # Not available in app
@bp.route('/api/alpha/user/change_password', methods=['PUT']) # Not available in app
@bp.route('/api/alpha/user/report_count', methods=['GET']) # Stage 2
@bp.route('/api/alpha/user/unread_count', methods=['GET']) # Stage 1
@bp.route('/api/alpha/user/verify_email', methods=['POST']) # Admin function. No plans to implement
@bp.route('/api/alpha/user/leave_admin', methods=['POST']) # Admin function. No plans to implement
@bp.route('/api/alpha/user/totp/generate', methods=['POST']) # Not available in app
@bp.route('/api/alpha/user/totp/update', methods=['POST']) # Not available in app
@bp.route('/api/alpha/user/export_settings', methods=['GET']) # Not available in app
@bp.route('/api/alpha/user/import_settings', methods=['POST']) # Not available in app
@bp.route('/api/alpha/user/list_logins', methods=['GET']) # Not available in app
@bp.route('/api/alpha/user/validate_auth', methods=['GET']) # Not available in app
@bp.route('/api/alpha/user/logout', methods=['POST']) # Stage 1
def alpha_user():
return jsonify({"error": "not_yet_implemented"}), 400
# Admin - not yet implemented
@bp.route('/api/alpha/admin/add', methods=['POST'])
@bp.route('/api/alpha/admin/registration_application/count', methods=['GET'])
@bp.route('/api/alpha/admin/registration_application/list', methods=['GET'])
@bp.route('/api/alpha/admin/registration_application/approve', methods=['PUT'])
@bp.route('/api/alpha/admin/purge/person', methods=['POST'])
@bp.route('/api/alpha/admin/purge/community', methods=['POST'])
@bp.route('/api/alpha/admin/purge/post', methods=['POST'])
@bp.route('/api/alpha/admin/purge/comment', methods=['POST'])
@bp.route('/api/alpha/post/like/list', methods=['GET'])
@bp.route('/api/alpha/comment/like/list', methods=['GET'])
@bp.route('/api/alpha/admin/registration_application/count', methods=['GET']) # (no
@bp.route('/api/alpha/admin/registration_application/list', methods=['GET']) # plans
@bp.route('/api/alpha/admin/registration_application/approve', methods=['PUT']) # to
@bp.route('/api/alpha/admin/purge/person', methods=['POST']) # implement
@bp.route('/api/alpha/admin/purge/community', methods=['POST']) # any
@bp.route('/api/alpha/admin/purge/post', methods=['POST']) # endpoints
@bp.route('/api/alpha/admin/purge/comment', methods=['POST']) # for
@bp.route('/api/alpha/post/like/list', methods=['GET']) # admin
@bp.route('/api/alpha/comment/like/list', methods=['GET']) # use)
def alpha_admin():
return jsonify({"error": "not_yet_implemented"}), 400
# CustomEmoji - not yet implemented
@bp.route('/api/alpha/custom_emoji', methods=['PUT'])
@bp.route('/api/alpha/custom_emoji', methods=['POST'])
@bp.route('/api/alpha/custom_emoji/delete', methods=['POST'])
@bp.route('/api/alpha/custom_emoji', methods=['PUT']) # (doesn't
@bp.route('/api/alpha/custom_emoji', methods=['POST']) # seem
@bp.route('/api/alpha/custom_emoji/delete', methods=['POST']) # important)
def alpha_emoji():
return jsonify({"error": "not_yet_implemented"}), 400

4
app/api/alpha/utils/__init__.py
View file

@ -1,7 +1,7 @@
from app.api.alpha.utils.site import get_site, post_site_block
from app.api.alpha.utils.misc import get_search
from app.api.alpha.utils.post import get_post_list, get_post, post_post_like, put_post_save, put_post_subscribe, post_post, put_post, post_post_delete, post_post_report
from app.api.alpha.utils.reply import get_reply_list, post_reply_like, put_reply_save, put_reply_subscribe, post_reply, put_reply, post_reply_delete, post_reply_report
from app.api.alpha.utils.post import get_post_list, get_post, post_post_like, put_post_save, put_post_subscribe, post_post, put_post, post_post_delete, post_post_report, post_post_lock, post_post_feature, post_post_remove
from app.api.alpha.utils.reply import get_reply_list, post_reply_like, put_reply_save, put_reply_subscribe, post_reply, put_reply, post_reply_delete, post_reply_report, post_reply_remove
from app.api.alpha.utils.community import get_community, get_community_list, post_community_follow, post_community_block
from app.api.alpha.utils.user import get_user, post_user_block

27
app/api/alpha/utils/community.py
View file

@ -6,33 +6,46 @@ from app.models import Community, CommunityMember
from app.shared.community import join_community, leave_community, block_community, unblock_community
from app.utils import communities_banned_from, blocked_instances
from sqlalchemy import desc
@cache.memoize(timeout=3)
def cached_community_list(type, user_id):
if type == 'Subscribed' and user_id is not None:
communities = Community.query.filter_by(banned=False).join(CommunityMember).filter(CommunityMember.user_id == user_id)
def cached_community_list(type, sort, limit, user_id):
if user_id:
banned_from = communities_banned_from(user_id)
if banned_from:
communities = communities.filter(Community.id.not_in(banned_from))
else:
banned_from = None
if type == 'Subscribed':
communities = Community.query.filter_by(banned=False).join(CommunityMember).filter(CommunityMember.user_id == user_id)
elif type == 'Local':
communities = Community.query.filter_by(ap_id=None, banned=False)
else:
communities = Community.query.filter_by(banned=False)
if user_id is not None:
if banned_from:
communities = communities.filter(Community.id.not_in(banned_from))
if user_id:
blocked_instance_ids = blocked_instances(user_id)
if blocked_instance_ids:
communities = communities.filter(Community.instance_id.not_in(blocked_instance_ids))
if sort == 'Active': # 'Trending Communities' screen
communities = communities.order_by(desc(Community.last_active)).limit(limit)
return communities.all()
def get_community_list(auth, data):
type = data['type_'] if data and 'type_' in data else "All"
sort = data['sort'] if data and 'sort' in data else "Hot"
page = int(data['page']) if data and 'page' in data else 1
limit = int(data['limit']) if data and 'limit' in data else 10
user_id = authorise_api_user(auth) if auth else None
communities = cached_community_list(type, user_id)
communities = cached_community_list(type, sort, limit, user_id)
start = (page - 1) * limit
end = start + limit

51
app/api/alpha/utils/post.py
View file

@ -3,7 +3,8 @@ from app.api.alpha.views import post_view, post_report_view
from app.api.alpha.utils.validators import required, integer_expected, boolean_expected, string_expected
from app.constants import POST_TYPE_ARTICLE, POST_TYPE_LINK, POST_TYPE_IMAGE, POST_TYPE_VIDEO
from app.models import Post, Community, CommunityMember, utcnow
from app.shared.post import vote_for_post, bookmark_the_post, remove_the_bookmark_from_post, toggle_post_notification, make_post, edit_post, delete_post, restore_post, report_post
from app.shared.post import vote_for_post, bookmark_the_post, remove_the_bookmark_from_post, toggle_post_notification, make_post, edit_post, \
delete_post, restore_post, report_post, lock_post, sticky_post, mod_remove_post, mod_restore_post
from app.utils import authorise_api_user, blocked_users, blocked_communities, blocked_instances, community_ids_from_instances, is_image_url, is_video_url
from datetime import timedelta
@ -239,3 +240,51 @@ def post_post_report(auth, data):
post_json = post_report_view(report=report, post_id=post_id, user_id=user_id)
return post_json
def post_post_lock(auth, data):
required(['post_id', 'locked'], data)
integer_expected(['post_id'], data)
boolean_expected(['locked'], data)
post_id = data['post_id']
locked = data['locked']
user_id, post = lock_post(post_id, locked, SRC_API, auth)
post_json = post_view(post=post, variant=4, user_id=user_id)
return post_json
def post_post_feature(auth, data):
required(['post_id', 'featured', 'feature_type'], data)
integer_expected(['post_id'], data)
boolean_expected(['featured'], data)
string_expected(['feature_type'], data)
post_id = data['post_id']
featured = data['featured']
user_id, post = sticky_post(post_id, featured, SRC_API, auth)
post_json = post_view(post=post, variant=4, user_id=user_id)
return post_json
def post_post_remove(auth, data):
required(['post_id', 'removed'], data)
integer_expected(['post_id'], data)
boolean_expected(['removed'], data)
string_expected(['reason'], data)
post_id = data['post_id']
removed = data['removed']
if removed == True:
reason = data['reason'] if 'reason' in data else 'Removed by mod'
user_id, post = mod_remove_post(post_id, reason, SRC_API, auth)
else:
reason = data['reason'] if 'reason' in data else 'Restored by mod'
user_id, post = mod_restore_post(post_id, reason, SRC_API, auth)
post_json = post_view(post=post, variant=4, user_id=user_id)
return post_json

21
app/api/alpha/utils/reply.py
View file

@ -3,7 +3,7 @@ from app.api.alpha.utils.validators import required, integer_expected, boolean_e
from app.api.alpha.views import reply_view, reply_report_view
from app.models import PostReply, Post
from app.shared.reply import vote_for_reply, bookmark_the_post_reply, remove_the_bookmark_from_post_reply, toggle_post_reply_notification, make_reply, edit_reply, \
delete_reply, restore_reply, report_reply
delete_reply, restore_reply, report_reply, mod_remove_reply, mod_restore_reply
from app.utils import authorise_api_user, blocked_users, blocked_instances
from sqlalchemy import desc
@ -191,3 +191,22 @@ def post_reply_report(auth, data):
reply_json = reply_report_view(report=report, reply_id=reply_id, user_id=user_id)
return reply_json
def post_reply_remove(auth, data):
required(['comment_id', 'removed'], data)
integer_expected(['comment_id'], data)
boolean_expected(['removed'], data)
string_expected(['reason'], data)
reply_id = data['comment_id']
removed = data['removed']
if removed == True:
reason = data['reason'] if 'reason' in data else 'Removed by mod'
user_id, reply = mod_remove_reply(reply_id, reason, SRC_API, auth)
else:
reason = data['reason'] if 'reason' in data else 'Restored by mod'
user_id, reply = mod_restore_reply(reply_id, reason, SRC_API, auth)
reply_json = reply_view(reply=reply, variant=4, user_id=user_id)
return reply_json

21
app/community/routes.py
View file

@ -21,8 +21,7 @@ from app.chat.util import send_message
from app.community.forms import SearchRemoteCommunity, CreateDiscussionForm, CreateImageForm, CreateLinkForm, \
ReportCommunityForm, \
DeleteCommunityForm, AddCommunityForm, EditCommunityForm, AddModeratorForm, BanUserCommunityForm, \
EscalateReportForm, ResolveReportForm, CreateVideoForm, CreatePollForm, RetrieveRemotePost, \
EditCommunityWikiPageForm
EscalateReportForm, ResolveReportForm, CreateVideoForm, CreatePollForm, EditCommunityWikiPageForm
from app.community.util import search_for_community, actor_to_community, \
save_icon_file, save_banner_file, send_to_remote_instance, \
delete_post_from_community, delete_post_reply_from_community, community_in_list, find_local_users, tags_from_string, \
@ -152,24 +151,6 @@ def add_remote():
site=g.site)
@bp.route('/retrieve_remote_post/<int:community_id>', methods=['GET', 'POST'])
@login_required
def retrieve_remote_post(community_id: int):
if current_user.banned:
return show_ban_message()
form = RetrieveRemotePost()
new_post = None
community = Community.query.get_or_404(community_id)
if form.validate_on_submit():
address = form.address.data.strip()
new_post = resolve_remote_post(address, community_id)
if new_post is None:
flash(_('Post not found.'), 'warning')
return render_template('community/retrieve_remote_post.html',
title=_('Retrieve Remote Post'), form=form, new_post=new_post, community=community)
# @bp.route('/c/<actor>', methods=['GET']) - defined in activitypub/routes.py, which calls this function for user requests. A bit weird.
def show_community(community: Community):

238
app/community/util.py
View file

@ -10,9 +10,9 @@ from flask_login import current_user
from pillow_heif import register_heif_opener
from app import db, cache, celery
from app.activitypub.signature import post_request, default_context
from app.activitypub.util import find_actor_or_create, actor_json_to_model, post_json_to_model, ensure_domains_match, \
find_hashtag_or_create
from app.activitypub.signature import post_request, default_context, signed_get_request
from app.activitypub.util import find_actor_or_create, actor_json_to_model, ensure_domains_match, \
find_hashtag_or_create, create_post, remote_object_to_json
from app.constants import POST_TYPE_ARTICLE, POST_TYPE_LINK, POST_TYPE_IMAGE, POST_TYPE_VIDEO, NOTIF_POST, \
POST_TYPE_POLL
from app.models import Community, File, BannedInstances, PostReply, Post, utcnow, CommunityMember, Site, \
@ -71,159 +71,129 @@ def search_for_community(address: str):
if community_json['type'] == 'Group':
community = actor_json_to_model(community_json, name, server)
if community:
if community.ap_profile_id == f"https://{server}/video-channels/{name}":
if current_app.debug:
retrieve_peertube_mods_and_backfill(community.id, community_json['attributedTo'])
else:
retrieve_peertube_mods_and_backfill.delay(community.id, community_json['attributedTo'])
return community
if current_app.debug:
retrieve_mods_and_backfill(community.id)
retrieve_mods_and_backfill(community.id, server, name, community_json)
else:
retrieve_mods_and_backfill.delay(community.id)
retrieve_mods_and_backfill.delay(community.id, server, name, community_json)
return community
return None
@celery.task
def retrieve_peertube_mods_and_backfill(community_id: int, mods: list):
community = Community.query.get(community_id)
site = Site.query.get(1)
for m in mods:
user = find_actor_or_create(m['id'])
if user:
existing_membership = CommunityMember.query.filter_by(community_id=community.id, user_id=user.id).first()
if existing_membership:
existing_membership.is_moderator = True
else:
new_membership = CommunityMember(community_id=community.id, user_id=user.id, is_moderator=True)
db.session.add(new_membership)
community.restricted_to_mods = True
db.session.commit()
if community.ap_public_url:
outbox_request = get_request(community.ap_outbox_url, headers={'Accept': 'application/activity+json'})
if outbox_request.status_code == 200:
outbox_data = outbox_request.json()
outbox_request.close()
if 'totalItems' in outbox_data and outbox_data['totalItems'] > 0:
page1_request = get_request(outbox_data['first'], headers={'Accept': 'application/activity+json'})
if page1_request.status_code == 200:
page1_data = page1_request.json()
page1_request.close()
if 'type' in page1_data and page1_data['type'] == 'OrderedCollectionPage' and 'orderedItems' in page1_data:
# only 10 posts per page for PeerTube
for activity in page1_data['orderedItems']:
video_request = get_request(activity['object'], headers={'Accept': 'application/activity+json'})
if video_request.status_code == 200:
video_data = video_request.json()
video_request.close()
activity_log = ActivityPubLog(direction='in', activity_id=video_data['id'], activity_type='Video', result='failure')
if site.log_activitypub_json:
activity_log.activity_json = json.dumps(video_data)
db.session.add(activity_log)
if not ensure_domains_match(video_data):
activity_log.exception_message = 'Domains do not match'
db.session.commit()
continue
if user and user.is_local():
activity_log.exception_message = 'Activity about local content which is already present'
db.session.commit()
continue
if user:
post = post_json_to_model(activity_log, video_data, user, community)
post.ap_announce_id = activity['id']
post.ranking = post.post_ranking(post.score, post.posted_at)
else:
activity_log.exception_message = 'Could not find or create actor'
db.session.commit()
if community.post_count > 0:
community.last_active = Post.query.filter(Post.community_id == community_id).order_by(desc(Post.posted_at)).first().posted_at
db.session.commit()
@celery.task
def retrieve_mods_and_backfill(community_id: int):
def retrieve_mods_and_backfill(community_id: int, server, name, community_json=None):
with current_app.app_context():
community = Community.query.get(community_id)
if not community:
return
site = Site.query.get(1)
if community.ap_moderators_url:
mods_request = get_request(community.ap_moderators_url, headers={'Accept': 'application/activity+json'})
if mods_request.status_code == 200:
mods_data = mods_request.json()
mods_request.close()
if mods_data and mods_data['type'] == 'OrderedCollection' and 'orderedItems' in mods_data:
for actor in mods_data['orderedItems']:
sleep(0.5)
user = find_actor_or_create(actor)
if user:
existing_membership = CommunityMember.query.filter_by(community_id=community.id, user_id=user.id).first()
is_peertube = is_guppe = is_wordpress = False
if community.ap_profile_id == f"https://{server}/video-channels/{name}":
is_peertube = True
elif community.ap_profile_id.startswith('https://a.gup.pe/u'):
is_guppe = True
# get mods
if community_json and 'attributedTo' in community_json:
mods = community_json['attributedTo']
if isinstance(mods, list):
for m in mods:
if 'type' in m and m['type'] == 'Person' and 'id' in m:
mod = find_actor_or_create(m['id'])
if mod:
existing_membership = CommunityMember.query.filter_by(community_id=community.id, user_id=mod.id).first()
if existing_membership:
existing_membership.is_moderator = True
else:
new_membership = CommunityMember(community_id=community.id, user_id=user.id, is_moderator=True)
new_membership = CommunityMember(community_id=community.id, user_id=mod.id, is_moderator=True)
db.session.add(new_membership)
db.session.commit()
elif community.ap_moderators_url:
mods_data = remote_object_to_json(community.ap_moderators_url)
if mods_data and mods_data['type'] == 'OrderedCollection' and 'orderedItems' in mods_data:
for actor in mods_data['orderedItems']:
sleep(0.5)
mod = find_actor_or_create(actor)
if mod:
existing_membership = CommunityMember.query.filter_by(community_id=community.id, user_id=mod.id).first()
if existing_membership:
existing_membership.is_moderator = True
else:
new_membership = CommunityMember(community_id=community.id, user_id=mod.id, is_moderator=True)
db.session.add(new_membership)
if is_peertube:
community.restricted_to_mods = True
db.session.commit()
# only backfill nsfw if nsfw communities are allowed
if (community.nsfw and not site.enable_nsfw) or (community.nsfl and not site.enable_nsfl):
return
# download 50 old posts
# download 50 old posts from unpaginated outboxes or 10 posts from page 1 if outbox is paginated (with Celery, or just 2 without)
if community.ap_outbox_url:
outbox_request = get_request(community.ap_outbox_url, headers={'Accept': 'application/activity+json'})
if outbox_request.status_code == 200:
outbox_data = outbox_request.json()
outbox_request.close()
if 'type' in outbox_data and outbox_data['type'] == 'OrderedCollection' and 'orderedItems' in outbox_data:
activities_processed = 0
for activity in outbox_data['orderedItems']:
activity_log = ActivityPubLog(direction='in', activity_id=activity['id'], activity_type='Announce', result='failure')
if site.log_activitypub_json:
activity_log.activity_json = json.dumps(activity)
db.session.add(activity_log)
if 'object' in activity and 'object' in activity['object']:
if not ensure_domains_match(activity['object']['object']):
activity_log.exception_message = 'Domains do not match'
db.session.commit()
continue
user = find_actor_or_create(activity['object']['actor'])
if user and user.is_local():
activity_log.exception_message = 'Activity about local content which is already present'
db.session.commit()
continue
if user:
post = post_json_to_model(activity_log, activity['object']['object'], user, community)
if post:
post.ap_create_id = activity['object']['id']
post.ap_announce_id = activity['id']
post.ranking = post.post_ranking(post.score, post.posted_at)
if post.url:
post.calculate_cross_posts()
db.session.commit()
else:
activity_log.exception_message = 'Could not find or create actor'
outbox_data = remote_object_to_json(community.ap_outbox_url)
if not outbox_data or ('totalItems' in outbox_data and outbox_data['totalItems'] == 0):
return
if 'first' in outbox_data:
outbox_data = remote_object_to_json(outbox_data['first'])
if not outbox_data:
return
max = 10
else:
max = 50
if current_app.debug:
max = 2
if 'type' in outbox_data and (outbox_data['type'] == 'OrderedCollection' or outbox_data['type'] == 'OrderedCollectionPage') and 'orderedItems' in outbox_data:
activities_processed = 0
for announce in outbox_data['orderedItems']:
activity = None
if is_peertube or is_guppe:
activity = remote_object_to_json(announce['object'])
elif 'object' in announce and 'object' in announce['object']:
activity = announce['object']['object']
elif 'type' in announce and announce['type'] == 'Create':
activity = announce['object']
is_wordpress = True
if not activity:
return
if not ensure_domains_match(activity):
continue
if is_peertube:
user = mod
elif 'attributedTo' in activity and isinstance(activity['attributedTo'], str):
user = find_actor_or_create(activity['attributedTo'])
if not user:
continue
else:
continue
if user.is_local():
continue
if is_peertube or is_guppe:
request_json = {'id': f"https://{server}/activities/create/{gibberish(15)}", 'object': activity}
elif is_wordpress:
request_json = announce
else:
request_json = announce['object']
post = create_post(True, community, request_json, user, announce['id'])
if post:
if 'published' in activity:
post.posted_at = activity['published']
post.last_active = activity['published']
db.session.commit()
activities_processed += 1
if activities_processed >= 50:
break
c = Community.query.get(community.id)
if c.post_count > 0:
c.last_active = Post.query.filter(Post.community_id == community_id).order_by(desc(Post.posted_at)).first().posted_at
activities_processed += 1
if activities_processed >= max:
break
if community.post_count > 0:
community.last_active = Post.query.filter(Post.community_id == community.id).order_by(desc(Post.posted_at)).first().posted_at
db.session.commit()
if community.ap_featured_url:
featured_request = get_request(community.ap_featured_url, headers={'Accept': 'application/activity+json'})
if featured_request.status_code == 200:
featured_data = featured_request.json()
featured_request.close()
if featured_data['type'] == 'OrderedCollection' and 'orderedItems' in featured_data:
for item in featured_data['orderedItems']:
featured_id = item['id']
p = Post.query.filter(Post.ap_id == featured_id).first()
if p:
p.sticky = True
db.session.commit()
if community.ap_featured_url:
featured_data = remote_object_to_json(community.ap_featured_url)
if featured_data and 'type' in featured_data and featured_data['type'] == 'OrderedCollection' and 'orderedItems' in featured_data:
for item in featured_data['orderedItems']:
featured_id = item['id']
p = Post.query.filter(Post.ap_id == featured_id).first()
if p:
p.sticky = True
db.session.commit()
def actor_to_community(actor) -> Community:

126
app/shared/post.py
View file

@ -7,7 +7,7 @@ from app.shared.tasks import task_selector
from app.utils import render_template, authorise_api_user, shorten_string, gibberish, ensure_directory_exists, \
piefed_markdown_to_lemmy_markdown, markdown_to_html, remove_tracking_from_link, domain_from_url, \
opengraph_parse, url_to_thumbnail_file, can_create_post, is_video_hosting_site, recently_upvoted_posts, \
is_image_url, is_video_hosting_site
is_image_url, is_video_hosting_site, add_to_modlog_activitypub
from flask import abort, flash, redirect, request, url_for, current_app, g
from flask_babel import _
@ -531,3 +531,127 @@ def report_post(post_id, input, src, auth=None):
return user_id, report
else:
return
def lock_post(post_id, locked, src, auth=None):
if src == SRC_API:
user = authorise_api_user(auth, return_type='model')
else:
user = current_user
post = Post.query.filter_by(id=post_id).one()
if locked:
comments_enabled = False
modlog_type = 'lock_post'
else:
comments_enabled = True
modlog_type = 'unlock_post'
if post.community.is_moderator(user) or post.community.is_instance_admin(user):
post.comments_enabled = comments_enabled
db.session.commit()
add_to_modlog_activitypub(modlog_type, user, community_id=post.community_id,
link_text=shorten_string(post.title), link=f'post/{post.id}', reason='')
if locked:
task_selector('lock_post', user_id=user.id, post_id=post_id)
else:
task_selector('unlock_post', user_id=user.id, post_id=post_id)
return user.id, post
def sticky_post(post_id, featured, src, auth=None):
if src == SRC_API:
user = authorise_api_user(auth, return_type='model')
else:
user = current_user
post = Post.query.filter_by(id=post_id).one()
community = post.community
if post.community.is_moderator(user) or post.community.is_instance_admin(user):
post.sticky = featured
if featured:
modlog_type = 'featured_post'
else:
modlog_type = 'unfeatured_post'
if not community.ap_featured_url:
community.ap_featured_url = community.ap_profile_id + '/featured'
db.session.commit()
add_to_modlog_activitypub(modlog_type, user, community_id=post.community_id,
link_text=shorten_string(post.title), link=f'post/{post.id}', reason='')
if featured:
task_selector('sticky_post', user_id=user.id, post_id=post_id)
else:
task_selector('unsticky_post', user_id=user.id, post_id=post_id)
return user.id, post
# mod deletes
def mod_remove_post(post_id, reason, src, auth):
if src == SRC_API:
user = authorise_api_user(auth, return_type='model')
else:
user = current_user
post = Post.query.filter_by(id=post_id, user_id=user.id, deleted=False).one()
if not post.community.is_moderator(user) and not post.community.is_instance_admin(user):
raise Exception('Does not have permission')
if post.url:
post.calculate_cross_posts(delete_only=True)
post.deleted = True
post.deleted_by = user.id
post.author.post_count -= 1
post.community.post_count -= 1
db.session.commit()
if src == SRC_WEB:
flash(_('Post deleted.'))
add_to_modlog_activitypub('delete_post', user, community_id=post.community_id,
link_text=shorten_string(post.title), link=f'post/{post.id}', reason=reason)
task_selector('delete_post', user_id=user.id, post_id=post.id, reason=reason)
if src == SRC_API:
return user.id, post
else:
return
def mod_restore_post(post_id, reason, src, auth):
if src == SRC_API:
user = authorise_api_user(auth, return_type='model')
else:
user = current_user
post = Post.query.filter_by(id=post_id, user_id=user.id, deleted=True).one()
if not post.community.is_moderator(user) and not post.community.is_instance_admin(user):
raise Exception('Does not have permission')
if post.url:
post.calculate_cross_posts()
post.deleted = False
post.deleted_by = None
post.author.post_count -= 1
post.community.post_count -= 1
db.session.commit()
if src == SRC_WEB:
flash(_('Post restored.'))
add_to_modlog_activitypub('restore_post', user, community_id=post.community_id,
link_text=shorten_string(post.title), link=f'post/{post.id}', reason=reason)
task_selector('restore_post', user_id=user.id, post_id=post.id, reason=reason)
if src == SRC_API:
return user.id, post
else:
return

65
app/shared/reply.py
View file

@ -5,7 +5,7 @@ from app.constants import *
from app.models import Instance, Notification, NotificationSubscription, Post, PostReply, PostReplyBookmark, Report, Site, User, utcnow
from app.shared.tasks import task_selector
from app.utils import gibberish, instance_banned, render_template, authorise_api_user, recently_upvoted_post_replies, recently_downvoted_post_replies, shorten_string, \
piefed_markdown_to_lemmy_markdown, markdown_to_html, ap_datetime
piefed_markdown_to_lemmy_markdown, markdown_to_html, ap_datetime, add_to_modlog_activitypub
from flask import abort, current_app, flash, redirect, request, url_for
from flask_babel import _
@ -337,3 +337,66 @@ def report_reply(reply_id, input, src, auth=None):
return user_id, report
else:
return
# mod deletes
def mod_remove_reply(reply_id, reason, src, auth):
if src == SRC_API:
user = authorise_api_user(auth, return_type='model')
else:
user = current_user
reply = PostReply.query.filter_by(id=reply_id, deleted=False).one()
if not reply.community.is_moderator(user) and not reply.community.is_instance_admin(user):
raise Exception('Does not have permission')
reply.deleted = True
reply.deleted_by = user.id
if not reply.author.bot:
reply.post.reply_count -= 1
reply.author.post_reply_count -= 1
db.session.commit()
if src == SRC_WEB:
flash(_('Comment deleted.'))
add_to_modlog_activitypub('delete_post_reply', user, community_id=reply.community_id,
link_text=shorten_string(f'comment on {shorten_string(reply.post.title)}'),
link=f'post/{reply.post_id}#comment_{reply.id}', reason=reason)
task_selector('delete_reply', user_id=user.id, reply_id=reply.id, reason=reason)
if src == SRC_API:
return user.id, reply
else:
return
def mod_restore_reply(reply_id, reason, src, auth):
if src == SRC_API:
user = authorise_api_user(auth, return_type='model')
else:
user = current_user
reply = PostReply.query.filter_by(id=reply_id, deleted=True).one()
if not reply.community.is_moderator(user) and not reply.community.is_instance_admin(user):
raise Exception('Does not have permission')
reply.deleted = False
reply.deleted_by = None
if not reply.author.bot:
reply.post.reply_count += 1
reply.author.post_reply_count += 1
db.session.commit()
if src == SRC_WEB:
flash(_('Comment restored.'))
add_to_modlog_activitypub('restore_post_reply', user, community_id=reply.community_id,
link_text=shorten_string(f'comment on {shorten_string(reply.post.title)}'),
link=f'post/{reply.post_id}#comment_{reply.id}', reason=reason)
task_selector('restore_reply', user_id=user.id, reply_id=reply.id, reason=reason)
if src == SRC_API:
return user.id, reply
else:
return

9
app/shared/tasks/__init__.py
View file

@ -4,6 +4,9 @@ from app.shared.tasks.notes import make_reply, edit_reply
from app.shared.tasks.deletes import delete_reply, restore_reply, delete_post, restore_post
from app.shared.tasks.flags import report_reply, report_post
from app.shared.tasks.pages import make_post, edit_post
from app.shared.tasks.locks import lock_post, unlock_post
from app.shared.tasks.adds import sticky_post
from app.shared.tasks.removes import unsticky_post
from flask import current_app
@ -23,7 +26,11 @@ def task_selector(task_key, send_async=True, **kwargs):
'edit_post': edit_post,
'delete_post': delete_post,
'restore_post': restore_post,
'report_post': report_post
'report_post': report_post,
'lock_post': lock_post,
'unlock_post': unlock_post,
'sticky_post': sticky_post,
'unsticky_post': unsticky_post
}
if current_app.debug:

82
app/shared/tasks/adds.py Normal file
View file

@ -0,0 +1,82 @@
from app import celery
from app.activitypub.signature import default_context, post_request
from app.models import Community, Post, User
from app.utils import gibberish, instance_banned
from flask import current_app
""" JSON format
Add:
{
'id':
'type':
'actor':
'object':
'target': (featured_url or moderators_url)
'@context':
'audience':
'to': []
'cc': []
}
For Announce, remove @context from inner object, and use same fields except audience
"""
@celery.task
def sticky_post(send_async, user_id, post_id):
post = Post.query.filter_by(id=post_id).one()
add_object(user_id, post)
@celery.task
def add_mod(send_async, user_id, mod_id, community_id):
mod = User.query.filter_by(id=mod_id).one()
add_object(user_id, mod, community_id)
def add_object(user_id, object, community_id=None):
user = User.query.filter_by(id=user_id).one()
if not community_id:
community = object.community
else:
community = Community.query.filter_by(id=community_id).one()
if community.local_only or not community.instance.online():
return
add_id = f"https://{current_app.config['SERVER_NAME']}/activities/add/{gibberish(15)}"
to = ["https://www.w3.org/ns/activitystreams#Public"]
cc = [community.public_url()]
add = {
'id': add_id,
'type': 'Add',
'actor': user.public_url(),
'object': object.public_url(),
'target': community.ap_moderators_url if community_id else community.ap_featured_url,
'@context': default_context(),
'audience': community.public_url(),
'to': to,
'cc': cc
}
if community.is_local():
del add['@context']
announce_id = f"https://{current_app.config['SERVER_NAME']}/activities/announce/{gibberish(15)}"
actor = community.public_url()
cc = [community.ap_followers_url]
announce = {
'id': announce_id,
'type': 'Announce',
'actor': actor,
'object': add,
'@context': default_context(),
'to': to,
'cc': cc
}
for instance in community.following_instances():
if instance.inbox and instance.online() and not user.has_blocked_instance(instance.id) and not instance_banned(instance.domain):
post_request(instance.inbox, announce, community.private_key, community.public_url() + '#main-key')
else:
post_request(community.ap_inbox_url, add, user.private_key, user.public_url() + '#main-key')

24
app/shared/tasks/deletes.py
View file

@ -13,6 +13,7 @@ Delete:
'type':
'actor':
'object':
'summary': (if deleted by mod / admin)
'@context':
'audience':
'to': []
@ -23,30 +24,30 @@ For Announce, remove @context from inner object, and use same fields except audi
@celery.task
def delete_reply(send_async, user_id, reply_id):
def delete_reply(send_async, user_id, reply_id, reason=None):
reply = PostReply.query.filter_by(id=reply_id).one()
delete_object(user_id, reply)
delete_object(user_id, reply, reason=reason)
@celery.task
def restore_reply(send_async, user_id, reply_id):
def restore_reply(send_async, user_id, reply_id, reason=None):
reply = PostReply.query.filter_by(id=reply_id).one()
delete_object(user_id, reply, is_restore=True)
delete_object(user_id, reply, is_restore=True, reason=reason)
@celery.task
def delete_post(send_async, user_id, post_id):
def delete_post(send_async, user_id, post_id, reason=None):
post = Post.query.filter_by(id=post_id).one()
delete_object(user_id, post, is_post=True)
delete_object(user_id, post, is_post=True, reason=reason)
@celery.task
def restore_post(send_async, user_id, post_id):
def restore_post(send_async, user_id, post_id, reason=None):
post = Post.query.filter_by(id=post_id).one()
delete_object(user_id, post, is_post=True, is_restore=True)
delete_object(user_id, post, is_post=True, is_restore=True, reason=reason)
def delete_object(user_id, object, is_post=False, is_restore=False):
def delete_object(user_id, object, is_post=False, is_restore=False, reason=None):
user = User.query.filter_by(id=user_id).one()
community = object.community
@ -81,6 +82,8 @@ def delete_object(user_id, object, is_post=False, is_restore=False):
'to': to,
'cc': cc
}
if reason:
delete['summary'] = reason
if is_restore:
del delete['@context']
@ -127,6 +130,9 @@ def delete_object(user_id, object, is_post=False, is_restore=False):
post_request(community.ap_inbox_url, payload, user.private_key, user.public_url() + '#main-key')
domains_sent_to.append(community.instance.domain)
if reason:
return
if is_post and followers:
payload = undo if is_restore else delete
for follower in followers:

97
app/shared/tasks/locks.py Normal file
View file

@ -0,0 +1,97 @@
from app import celery
from app.activitypub.signature import default_context, post_request
from app.models import Post, User
from app.utils import gibberish, instance_banned
from flask import current_app
""" JSON format
Lock:
{
'id':
'type':
'actor':
'object':
'@context':
'audience':
'to': []
'cc': []
}
For Announce, remove @context from inner object, and use same fields except audience
"""
@celery.task
def lock_post(send_async, user_id, post_id):
post = Post.query.filter_by(id=post_id).one()
lock_object(user_id, post)
@celery.task
def unlock_post(send_async, user_id, post_id):
post = Post.query.filter_by(id=post_id).one()
lock_object(user_id, post, is_undo=True)
def lock_object(user_id, object, is_undo=False):
user = User.query.filter_by(id=user_id).one()
community = object.community
if community.local_only or not community.instance.online():
return
lock_id = f"https://{current_app.config['SERVER_NAME']}/activities/lock/{gibberish(15)}"
to = ["https://www.w3.org/ns/activitystreams#Public"]
cc = [community.public_url()]
lock = {
'id': lock_id,
'type': 'Lock',
'actor': user.public_url(),
'object': object.public_url(),
'@context': default_context(),
'audience': community.public_url(),
'to': to,
'cc': cc
}
if is_undo:
del lock['@context']
undo_id = f"https://{current_app.config['SERVER_NAME']}/activities/undo/{gibberish(15)}"
undo = {
'id': undo_id,
'type': 'Undo',
'actor': user.public_url(),
'object': lock,
'@context': default_context(),
'audience': community.public_url(),
'to': to,
'cc': cc
}
if community.is_local():
if is_undo:
del undo['@context']
object=undo
else:
del lock['@context']
object=lock
announce_id = f"https://{current_app.config['SERVER_NAME']}/activities/announce/{gibberish(15)}"
actor = community.public_url()
cc = [community.ap_followers_url]
announce = {
'id': announce_id,
'type': 'Announce',
'actor': actor,
'object': object,
'@context': default_context(),
'to': to,
'cc': cc
}
for instance in community.following_instances():
if instance.inbox and instance.online() and not user.has_blocked_instance(instance.id) and not instance_banned(instance.domain):
post_request(instance.inbox, announce, community.private_key, community.public_url() + '#main-key')
else:
payload = undo if is_undo else lock
post_request(community.ap_inbox_url, payload, user.private_key, user.public_url() + '#main-key')

82
app/shared/tasks/removes.py Normal file
View file

@ -0,0 +1,82 @@
from app import celery
from app.activitypub.signature import default_context, post_request
from app.models import Community, Post, User
from app.utils import gibberish, instance_banned
from flask import current_app
""" JSON format
Remove:
{
'id':
'type':
'actor':
'object':
'target': (featured_url or moderators_url)
'@context':
'audience':
'to': []
'cc': []
}
For Announce, remove @context from inner object, and use same fields except audience
"""
@celery.task
def unsticky_post(send_async, user_id, post_id):
post = Post.query.filter_by(id=post_id).one()
remove_object(user_id, post)
@celery.task
def remove_mod(send_async, user_id, mod_id, community_id):
mod = User.query.filter_by(id=mod_id).one()
remove_object(user_id, mod, community_id)
def remove_object(user_id, object, community_id=None):
user = User.query.filter_by(id=user_id).one()
if not community_id:
community = object.community
else:
community = Community.query.filter_by(id=community_id).one()
if community.local_only or not community.instance.online():
return
remove_id = f"https://{current_app.config['SERVER_NAME']}/activities/remove/{gibberish(15)}"
to = ["https://www.w3.org/ns/activitystreams#Public"]
cc = [community.public_url()]
remove = {
'id': remove_id,
'type': 'Remove',
'actor': user.public_url(),
'object': object.public_url(),
'target': community.ap_moderators_url if community_id else community.ap_featured_url,
'@context': default_context(),
'audience': community.public_url(),
'to': to,
'cc': cc
}
if community.is_local():
del remove['@context']
announce_id = f"https://{current_app.config['SERVER_NAME']}/activities/announce/{gibberish(15)}"
actor = community.public_url()
cc = [community.ap_followers_url]
announce = {
'id': announce_id,
'type': 'Announce',
'actor': actor,
'object': remove,
'@context': default_context(),
'to': to,
'cc': cc
}
for instance in community.following_instances():
if instance.inbox and instance.online() and not user.has_blocked_instance(instance.id) and not instance_banned(instance.domain):
post_request(instance.inbox, announce, community.private_key, community.public_url() + '#main-key')
else:
post_request(community.ap_inbox_url, remove, user.private_key, user.public_url() + '#main-key')

2
app/templates/_side_pane.html
View file

@ -52,7 +52,7 @@
{% if rss_feed and not community.is_local() -%}
<ul>
<li><p><a href="{{ community.public_url() }}">{{ _('View community on original server') }}</a></p></li>
<li><p><a href="{{ url_for('community.retrieve_remote_post', community_id=community.id) }}">{{ _('Retrieve a post from the original server') }}</a></p></li>
<li><p><a href="{{ url_for('search.retrieve_remote_post') }}">{{ _('Retrieve a post from the original server') }}</a></p></li>
</ul>
{% endif -%}
{% if community.local_only -%}