From 0cab6461955eaf1523e9e73a42629cf687449662 Mon Sep 17 00:00:00 2001 From: freamon Date: Tue, 14 Jan 2025 13:54:17 +0000 Subject: [PATCH 01/19] API: initial support for new posts (polls and uploaded images not supported yet) --- app/api/alpha/routes.py | 109 +++++++------ app/api/alpha/utils/__init__.py | 2 +- app/api/alpha/utils/post.py | 37 ++++- app/shared/post.py | 119 +++++++++++++- app/shared/tasks/__init__.py | 5 +- app/shared/tasks/pages.py | 266 ++++++++++++++++++++++++++++++++ 6 files changed, 482 insertions(+), 56 deletions(-) create mode 100644 app/shared/tasks/pages.py diff --git a/app/api/alpha/routes.py b/app/api/alpha/routes.py index 5d274d63..22eadfcf 100644 --- a/app/api/alpha/routes.py +++ b/app/api/alpha/routes.py @@ -1,7 +1,7 @@ from app.api.alpha import bp from app.api.alpha.utils import get_site, post_site_block, \ get_search, \ - get_post_list, get_post, post_post_like, put_post_save, put_post_subscribe, \ + get_post_list, get_post, post_post_like, put_post_save, put_post_subscribe, post_post, \ get_reply_list, post_reply_like, put_reply_save, put_reply_subscribe, post_reply, put_reply, post_reply_delete, post_reply_report, \ get_community_list, get_community, post_community_follow, post_community_block, \ get_user, post_user_block @@ -9,12 +9,14 @@ from app.shared.auth import log_user_in from flask import current_app, jsonify, request +def enable_api(): + return True if current_app.debug else False # Site @bp.route('/api/alpha/site', methods=['GET']) def get_alpha_site(): - if not current_app.debug: - return jsonify({'error': 'alpha api routes only available in debug mode'}) + if not enable_api(): + return jsonify({'error': 'alpha api is not enabled'}) try: auth = request.headers.get('Authorization') return jsonify(get_site(auth)) @@ -24,8 +26,8 @@ def get_alpha_site(): @bp.route('/api/alpha/site/block', methods=['POST']) def get_alpha_site_block(): - if not current_app.debug: - return jsonify({'error': 'alpha api routes only available in debug mode'}) + if not enable_api(): + return jsonify({'error': 'alpha api is not enabled'}) try: auth = request.headers.get('Authorization') data = request.get_json(force=True) or {} @@ -37,8 +39,8 @@ def get_alpha_site_block(): # Misc @bp.route('/api/alpha/search', methods=['GET']) def get_alpha_search(): - if not current_app.debug: - return jsonify({'error': 'alpha api routes only available in debug mode'}) + if not enable_api(): + return jsonify({'error': 'alpha api is not enabled'}) try: auth = request.headers.get('Authorization') data = request.args.to_dict() or None @@ -50,8 +52,8 @@ def get_alpha_search(): # Community @bp.route('/api/alpha/community', methods=['GET']) def get_alpha_community(): - if not current_app.debug: - return jsonify({'error': 'alpha api routes only available in debug mode'}) + if not enable_api(): + return jsonify({'error': 'alpha api is not enabled'}) try: auth = request.headers.get('Authorization') data = request.args.to_dict() or None @@ -62,8 +64,8 @@ def get_alpha_community(): @bp.route('/api/alpha/community/list', methods=['GET']) def get_alpha_community_list(): - if not current_app.debug: - return jsonify({'error': 'alpha api routes only available in debug mode'}) + if not enable_api(): + return jsonify({'error': 'alpha api is not enabled'}) try: auth = request.headers.get('Authorization') data = request.args.to_dict() or None @@ -74,8 +76,8 @@ def get_alpha_community_list(): @bp.route('/api/alpha/community/follow', methods=['POST']) def post_alpha_community_follow(): - if not current_app.debug: - return jsonify({'error': 'alpha api routes only available in debug mode'}) + if not enable_api(): + return jsonify({'error': 'alpha api is not enabled'}) try: auth = request.headers.get('Authorization') data = request.get_json(force=True) or {} @@ -86,8 +88,8 @@ def post_alpha_community_follow(): @bp.route('/api/alpha/community/block', methods=['POST']) def post_alpha_community_block(): - if not current_app.debug: - return jsonify({'error': 'alpha api routes only available in debug mode'}) + if not enable_api(): + return jsonify({'error': 'alpha api is not enabled'}) try: auth = request.headers.get('Authorization') data = request.get_json(force=True) or {} @@ -99,8 +101,8 @@ def post_alpha_community_block(): # Post @bp.route('/api/alpha/post/list', methods=['GET']) def get_alpha_post_list(): - if not current_app.debug: - return jsonify({'error': 'alpha api routes only available in debug mode'}) + if not enable_api(): + return jsonify({'error': 'alpha api is not enabled'}) try: auth = request.headers.get('Authorization') data = request.args.to_dict() or None @@ -111,8 +113,8 @@ def get_alpha_post_list(): @bp.route('/api/alpha/post', methods=['GET']) def get_alpha_post(): - if not current_app.debug: - return jsonify({'error': 'alpha api routes only available in debug mode'}) + if not enable_api(): + return jsonify({'error': 'alpha api is not enabled'}) try: auth = request.headers.get('Authorization') data = request.args.to_dict() or None @@ -123,8 +125,8 @@ def get_alpha_post(): @bp.route('/api/alpha/post/like', methods=['POST']) def post_alpha_post_like(): - if not current_app.debug: - return jsonify({'error': 'alpha api routes only available in debug mode'}) + if not enable_api(): + return jsonify({'error': 'alpha api is not enabled'}) try: auth = request.headers.get('Authorization') data = request.get_json(force=True) or {} @@ -135,8 +137,8 @@ def post_alpha_post_like(): @bp.route('/api/alpha/post/save', methods=['PUT']) def put_alpha_post_save(): - if not current_app.debug: - return jsonify({'error': 'alpha api routes only available in debug mode'}) + if not enable_api(): + return jsonify({'error': 'alpha api is not enabled'}) try: auth = request.headers.get('Authorization') data = request.get_json(force=True) or {} @@ -147,8 +149,8 @@ def put_alpha_post_save(): @bp.route('/api/alpha/post/subscribe', methods=['PUT']) def put_alpha_post_subscribe(): - if not current_app.debug: - return jsonify({'error': 'alpha api routes only available in debug mode'}) + if not enable_api(): + return jsonify({'error': 'alpha api is not enabled'}) try: auth = request.headers.get('Authorization') data = request.get_json(force=True) or {} @@ -157,11 +159,23 @@ def put_alpha_post_subscribe(): return jsonify({"error": str(ex)}), 400 +@bp.route('/api/alpha/post', methods=['POST']) +def post_alpha_post(): + if not enable_api(): + return jsonify({'error': 'alpha api is not enabled'}) + try: + auth = request.headers.get('Authorization') + data = request.get_json(force=True) or {} + return jsonify(post_post(auth, data)) + except Exception as ex: + return jsonify({"error": str(ex)}), 400 + + # Reply @bp.route('/api/alpha/comment/list', methods=['GET']) def get_alpha_comment_list(): - if not current_app.debug: - return jsonify({'error': 'alpha api routes only available in debug mode'}) + if not enable_api(): + return jsonify({'error': 'alpha api is not enabled'}) try: auth = request.headers.get('Authorization') data = request.args.to_dict() or None @@ -172,8 +186,8 @@ def get_alpha_comment_list(): @bp.route('/api/alpha/comment/like', methods=['POST']) def post_alpha_comment_like(): - if not current_app.debug: - return jsonify({'error': 'alpha api routes only available in debug mode'}) + if not enable_api(): + return jsonify({'error': 'alpha api is not enabled'}) try: auth = request.headers.get('Authorization') data = request.get_json(force=True) or {} @@ -184,8 +198,8 @@ def post_alpha_comment_like(): @bp.route('/api/alpha/comment/save', methods=['PUT']) def put_alpha_comment_save(): - if not current_app.debug: - return jsonify({'error': 'alpha api routes only available in debug mode'}) + if not enable_api(): + return jsonify({'error': 'alpha api is not enabled'}) try: auth = request.headers.get('Authorization') data = request.get_json(force=True) or {} @@ -196,8 +210,8 @@ def put_alpha_comment_save(): @bp.route('/api/alpha/comment/subscribe', methods=['PUT']) def put_alpha_comment_subscribe(): - if not current_app.debug: - return jsonify({'error': 'alpha api routes only available in debug mode'}) + if not enable_api(): + return jsonify({'error': 'alpha api is not enabled'}) try: auth = request.headers.get('Authorization') data = request.get_json(force=True) or {} @@ -208,8 +222,8 @@ def put_alpha_comment_subscribe(): @bp.route('/api/alpha/comment', methods=['POST']) def post_alpha_comment(): - if not current_app.debug: - return jsonify({'error': 'alpha api routes only available in debug mode'}) + if not enable_api(): + return jsonify({'error': 'alpha api is not enabled'}) try: auth = request.headers.get('Authorization') data = request.get_json(force=True) or {} @@ -220,8 +234,8 @@ def post_alpha_comment(): @bp.route('/api/alpha/comment', methods=['PUT']) def put_alpha_comment(): - if not current_app.debug: - return jsonify({'error': 'alpha api routes only available in debug mode'}) + if not enable_api(): + return jsonify({'error': 'alpha api is not enabled'}) try: auth = request.headers.get('Authorization') data = request.get_json(force=True) or {} @@ -232,8 +246,8 @@ def put_alpha_comment(): @bp.route('/api/alpha/comment/delete', methods=['POST']) def post_alpha_comment_delete(): - if not current_app.debug: - return jsonify({'error': 'alpha api routes only available in debug mode'}) + if not enable_api(): + return jsonify({'error': 'alpha api is not enabled'}) try: auth = request.headers.get('Authorization') data = request.get_json(force=True) or {} @@ -244,8 +258,8 @@ def post_alpha_comment_delete(): @bp.route('/api/alpha/comment/report', methods=['POST']) def post_alpha_comment_report(): - if not current_app.debug: - return jsonify({'error': 'alpha api routes only available in debug mode'}) + if not enable_api(): + return jsonify({'error': 'alpha api is not enabled'}) try: auth = request.headers.get('Authorization') data = request.get_json(force=True) or {} @@ -257,8 +271,8 @@ def post_alpha_comment_report(): # User @bp.route('/api/alpha/user', methods=['GET']) def get_alpha_user(): - if not current_app.debug: - return jsonify({'error': 'alpha api routes only available in debug mode'}) + if not enable_api(): + return jsonify({'error': 'alpha api is not enabled'}) try: auth = request.headers.get('Authorization') data = request.args.to_dict() or None @@ -269,8 +283,8 @@ def get_alpha_user(): @bp.route('/api/alpha/user/login', methods=['POST']) def post_alpha_user_login(): - if not current_app.debug: - return jsonify({'error': 'alpha api routes only available in debug mode'}) + if not enable_api(): + return jsonify({'error': 'alpha api is not enabled'}) try: SRC_API = 3 # would be in app.constants data = request.get_json(force=True) or {} @@ -281,8 +295,8 @@ def post_alpha_user_login(): @bp.route('/api/alpha/user/block', methods=['POST']) def post_alpha_user_block(): - if not current_app.debug: - return jsonify({'error': 'alpha api routes only available in debug mode'}) + if not enable_api(): + return jsonify({'error': 'alpha api is not enabled'}) try: auth = request.headers.get('Authorization') data = request.get_json(force=True) or {} @@ -320,7 +334,6 @@ def alpha_community(): # Post - not yet implemented @bp.route('/api/alpha/post', methods=['PUT']) -@bp.route('/api/alpha/post', methods=['POST']) @bp.route('/api/alpha/post/delete', methods=['POST']) @bp.route('/api/alpha/post/remove', methods=['POST']) @bp.route('/api/alpha/post/lock', methods=['POST']) diff --git a/app/api/alpha/utils/__init__.py b/app/api/alpha/utils/__init__.py index 824050da..e43dd700 100644 --- a/app/api/alpha/utils/__init__.py +++ b/app/api/alpha/utils/__init__.py @@ -1,6 +1,6 @@ from app.api.alpha.utils.site import get_site, post_site_block from app.api.alpha.utils.misc import get_search -from app.api.alpha.utils.post import get_post_list, get_post, post_post_like, put_post_save, put_post_subscribe +from app.api.alpha.utils.post import get_post_list, get_post, post_post_like, put_post_save, put_post_subscribe, post_post from app.api.alpha.utils.reply import get_reply_list, post_reply_like, put_reply_save, put_reply_subscribe, post_reply, put_reply, post_reply_delete, post_reply_report from app.api.alpha.utils.community import get_community, get_community_list, post_community_follow, post_community_block from app.api.alpha.utils.user import get_user, post_user_block diff --git a/app/api/alpha/utils/post.py b/app/api/alpha/utils/post.py index 0d9fcf32..fd141057 100644 --- a/app/api/alpha/utils/post.py +++ b/app/api/alpha/utils/post.py @@ -1,9 +1,9 @@ from app import cache from app.api.alpha.views import post_view -from app.api.alpha.utils.validators import required, integer_expected, boolean_expected +from app.api.alpha.utils.validators import required, integer_expected, boolean_expected, string_expected from app.models import Post, Community, CommunityMember, utcnow -from app.shared.post import vote_for_post, bookmark_the_post, remove_the_bookmark_from_post, toggle_post_notification -from app.utils import authorise_api_user, blocked_users, blocked_communities, blocked_instances, community_ids_from_instances +from app.shared.post import vote_for_post, bookmark_the_post, remove_the_bookmark_from_post, toggle_post_notification, make_post +from app.utils import authorise_api_user, blocked_users, blocked_communities, blocked_instances, community_ids_from_instances, is_image_url from datetime import timedelta from sqlalchemy import desc @@ -149,3 +149,34 @@ def put_post_subscribe(auth, data): post_json = post_view(post=post_id, variant=4, user_id=user_id) return post_json + +def post_post(auth, data): + required(['title', 'community_id'], data) + integer_expected(['language_id'], data) + boolean_expected(['nsfw'], data) + string_expected(['string', 'body'], data) + + title = data['title'] + community_id = data['community_id'] + body = data['body'] if 'body' in data else '' + url = data['url'] if 'url' in data else None + nsfw = data['nsfw'] if 'nsfw' in data else False + language_id = data['language_id'] if 'language_id' in data else 2 # FIXME: use site language + if language_id < 2: + language_id = 2 + + if not url: + type = 'discussion' + elif is_image_url(url): + type = 'image' + else: + type = 'link' + + input = {'title': title, 'body': body, 'url': url, 'nsfw': nsfw, 'language_id': language_id, 'notify_author': True} + community = Community.query.filter_by(id=community_id).one() + user_id, post = make_post(input, community, type, SRC_API, auth) + + post_json = post_view(post=post, variant=4, user_id=user_id) + return post_json + + diff --git a/app/shared/post.py b/app/shared/post.py index 3df03a70..31e39d53 100644 --- a/app/shared/post.py +++ b/app/shared/post.py @@ -1,13 +1,18 @@ from app import db from app.constants import * -from app.models import NotificationSubscription, Post, PostBookmark +from app.community.util import tags_from_string +from app.models import Language, NotificationSubscription, Post, PostBookmark from app.shared.tasks import task_selector -from app.utils import render_template, authorise_api_user, shorten_string +from app.utils import render_template, authorise_api_user, shorten_string, gibberish, ensure_directory_exists -from flask import abort, flash, redirect, request, url_for +from flask import abort, flash, redirect, request, url_for, current_app from flask_babel import _ from flask_login import current_user +from pillow_heif import register_heif_opener +from PIL import Image, ImageOps + +import os # would be in app/constants.py SRC_WEB = 1 @@ -127,3 +132,111 @@ def toggle_post_notification(post_id: int, src, auth=None): return user_id else: return render_template('post/_post_notification_toggle.html', post=post) + + +def make_post(input, community, type, src, auth=None, uploaded_file=None): + if src == SRC_API: + user = authorise_api_user(auth, return_type='model') + #if not basic_rate_limit_check(user): + # raise Exception('rate_limited') + title = input['title'] + body = input['body'] + url = input['url'] + nsfw = input['nsfw'] + notify_author = input['notify_author'] + language_id = input['language_id'] + tags = [] + else: + user = current_user + title = input.title.data + body = input.body.data + url = input.link_url.data + nsfw = input.nsfw.data + notify_author = input.notify_author.data + language_id = input.language_id.data + tags = tags_from_string(input.tags.data) + + language = Language.query.filter_by(id=language_id).one() + + request_json = { + 'id': None, + 'object': { + 'name': title, + 'type': 'Page', + 'stickied': False if src == SRC_API else input.sticky.data, + 'sensitive': nsfw, + 'nsfl': False if src == SRC_API else input.nsfl.data, + 'id': gibberish(), # this will be updated once we have the post.id + 'mediaType': 'text/markdown', + 'content': body, + 'tag': tags, + 'language': {'identifier': language.code, 'name': language.name} + } + } + + if type == 'link' or (type == 'image' and src == SRC_API): + request_json['object']['attachment'] = [{'type': 'Link', 'href': url}] + elif type == 'image' and src == SRC_WEB and uploaded_file and uploaded_file.filename != '': + # check if this is an allowed type of file + file_ext = os.path.splitext(uploaded_file.filename)[1] + if file_ext.lower() not in allowed_extensions: + abort(400, description="Invalid image type.") + + new_filename = gibberish(15) + # set up the storage directory + directory = 'app/static/media/posts/' + new_filename[0:2] + '/' + new_filename[2:4] + ensure_directory_exists(directory) + + final_place = os.path.join(directory, new_filename + file_ext) + uploaded_file.seek(0) + uploaded_file.save(final_place) + + if file_ext.lower() == '.heic': + register_heif_opener() + if file_ext.lower() == '.avif': + import pillow_avif + + Image.MAX_IMAGE_PIXELS = 89478485 + + # resize if necessary + if not final_place.endswith('.svg'): + img = Image.open(final_place) + if '.' + img.format.lower() in allowed_extensions: + img = ImageOps.exif_transpose(img) + + # limit full sized version to 2000px + img.thumbnail((2000, 2000)) + img.save(final_place) + + request_json['object']['attachment'] = [{ + 'type': 'Image', + 'url': f'https://{current_app.config["SERVER_NAME"]}/{final_place.replace("app/", "")}', + 'name': input.image_alt_text.data, + 'file_path': final_place + }] + elif type == 'video': + request_json['object']['attachment'] = [{'type': 'Document', 'url': url}] + elif type == 'poll': + request_json['object']['type'] = 'Question' + choices = [input.choice_1, input.choice_2, input.choice_3, input.choice_4, input.choice_5, + input.choice_6, input.choice_7, input.choice_8, input.choice_9, input.choice_10] + key = 'oneOf' if input.mode.data == 'single' else 'anyOf' + request_json['object'][key] = [] + for choice in choices: + choice_data = choice.data.strip() + if choice_data: + request_json['object'][key].append({'name': choice_data}) + request_json['object']['endTime'] = end_poll_date(input.finish_in.data) + + post = Post.new(user, community, request_json) + post.ap_id = f"https://{current_app.config['SERVER_NAME']}/post/{post.id}" + db.session.commit() + + task_selector('make_post', user_id=user.id, post_id=post.id) + + if src == SRC_API: + return user.id, post + else: + return post + + diff --git a/app/shared/tasks/__init__.py b/app/shared/tasks/__init__.py index 3c0a980a..8605a5b0 100644 --- a/app/shared/tasks/__init__.py +++ b/app/shared/tasks/__init__.py @@ -3,6 +3,7 @@ from app.shared.tasks.likes import vote_for_post, vote_for_reply from app.shared.tasks.notes import make_reply, edit_reply from app.shared.tasks.deletes import delete_reply, restore_reply from app.shared.tasks.flags import report_reply +from app.shared.tasks.pages import make_post, edit_post from flask import current_app @@ -17,7 +18,9 @@ def task_selector(task_key, send_async=True, **kwargs): 'edit_reply': edit_reply, 'delete_reply': delete_reply, 'restore_reply': restore_reply, - 'report_reply': report_reply + 'report_reply': report_reply, + 'make_post': make_post, + 'edit_post': edit_post } if current_app.debug: diff --git a/app/shared/tasks/pages.py b/app/shared/tasks/pages.py new file mode 100644 index 00000000..0499d59e --- /dev/null +++ b/app/shared/tasks/pages.py @@ -0,0 +1,266 @@ +from app import celery, db +from app.activitypub.signature import default_context, post_request +from app.constants import POST_TYPE_LINK, POST_TYPE_ARTICLE, POST_TYPE_IMAGE, POST_TYPE_VIDEO, POST_TYPE_POLL, MICROBLOG_APPS +from app.models import CommunityBan, Instance, Notification, Post, User, UserFollower, utcnow +from app.user.utils import search_for_user +from app.utils import gibberish, instance_banned, ap_datetime + +from flask import current_app +from flask_babel import _ + +import re + + +""" Post JSON format +{ + 'id': + 'type': + 'attributedTo': + 'to': [] + 'cc': [] + 'tag': [] + 'audience': + 'content': + 'mediaType': + 'source': {} + 'published': + 'updated': (inner oject of Update only) + 'language': {} + 'name': (not included in Polls, which are sent out as microblogs) + 'attachment': [] + 'commentsEnabled': + 'sensitive': + 'nsfl': + 'stickied': + 'image': (for posts with thumbnails only) + 'endTime': (last 3 are for polls) + 'votersCount': + 'oneOf' / 'anyOf': +} +""" +""" Create / Update / Announce JSON format +{ + 'id': + 'type': + 'actor': + 'object': + 'to': [] + 'cc': [] + '@context': (outer object only) + 'audience': (not in Announce) + 'tag': [] (not in Announce) +} +""" + + + +@celery.task +def make_post(send_async, user_id, post_id): + send_post(user_id, post_id) + + +@celery.task +def edit_post(send_async, user_id, post_id): + send_post(user_id, post_id, edit=True) + + +def send_post(user_id, post_id, edit=False): + user = User.query.filter_by(id=user_id).one() + post = Post.query.filter_by(id=post_id).one() + community = post.community + + # Find any users Mentioned in post body with @user@instance syntax + recipients = [] + pattern = r"@([a-zA-Z0-9_.-]*)@([a-zA-Z0-9_.-]*)\b" + matches = re.finditer(pattern, post.body) + for match in matches: + recipient = None + if match.group(2) == current_app.config['SERVER_NAME']: + user_name = match.group(1) + if user_name != user.user_name: + try: + recipient = search_for_user(user_name) + except: + pass + else: + ap_id = f"{match.group(1)}@{match.group(2)}" + try: + recipient = search_for_user(ap_id) + except: + pass + if recipient: + add_recipient = True + for existing_recipient in recipients: + if ((not recipient.ap_id and recipient.user_name == existing_recipient.user_name) or + (recipient.ap_id and recipient.ap_id == existing_recipient.ap_id)): + add_recipient = False + break + if add_recipient: + recipients.append(recipient) + + # Notify any local users that have been Mentioned + for recipient in recipients: + if recipient.is_local(): + if edit: + existing_notification = Notification.query.filter(Notification.user_id == recipient.id, Notification.url == f"https://{current_app.config['SERVER_NAME']}/post/{post.id}").first() + else: + existing_notification = None + if not existing_notification: + notification = Notification(user_id=recipient.id, title=_(f"You have been mentioned in post {post.id}"), + url=f"https://{current_app.config['SERVER_NAME']}/post/{post.id}", + author_id=user.id) + recipient.unread_notifications += 1 + db.session.add(notification) + db.session.commit() + + if community.local_only or not community.instance.online(): + return + + banned = CommunityBan.query.filter_by(user_id=user_id, community_id=community.id).first() + if banned: + return + if not community.is_local(): + if user.has_blocked_instance(community.instance.id) or instance_banned(community.instance.domain): + return + + type = 'Question' if post.type == POST_TYPE_POLL else 'Page' + to = [community.public_url(), "https://www.w3.org/ns/activitystreams#Public"] + cc = [] + tag = post.tags_for_activitypub() + for recipient in recipients: + tag.append({'href': recipient.public_url(), 'name': recipient.mention_tag(), 'type': 'Mention'}) + cc.append(recipient.public_url()) + language = {'identifier': post.language_code(), 'name': post.language_name()} + source = {'content': post.body, 'mediaType': 'text/markdown'} + attachment = [] + if post.type == POST_TYPE_LINK or post.type == POST_TYPE_VIDEO: + attachment.append({'href': post.url, 'type': 'Link'}) + elif post.type == POST_TYPE_IMAGE: + attachment.append({'type': 'Image', 'url': post.image.source_url, 'name': post.image.alt_text}) + + page = { + 'id': post.public_url(), + 'type': type, + 'attributedTo': user.public_url(), + 'to': to, + 'cc': cc, + 'tag': tag, + 'audience': community.public_url(), + 'content': post.body_html if post.type != POST_TYPE_POLL else '

' + post.title + '

', + 'mediaType': 'text/html', + 'source': source, + 'published': ap_datetime(post.posted_at), + 'language': language, + 'name': post.title, + 'attachment': attachment, + 'commentsEnabled': post.comments_enabled, + 'sensitive': post.nsfw or post.nsfl, + 'nsfl': post.nsfl, + 'stickied': post.sticky + } + if post.type != POST_TYPE_POLL: + page['name'] = post.title + if edit: + page['updated']: ap_datetime(utcnow()) + if post.image_id: + image_url = '' + if post.image.source_url: + image_url = post.image.source_url + elif post.image.file_path: + image_url = post.image.file_path.replace('app/static/', f"https://{current_app.config['SERVER_NAME']}/static/") + elif post.image.thumbnail_path: + image_url = post.image.thumbnail_path.replace('app/static/', f"https://{current_app.config['SERVER_NAME']}/static/") + page['image'] = {'type': 'Image', 'url': image_url} + if post.type == POST_TYPE_POLL: + poll = Poll.query.filter_by(post_id=post.id).first() + page['endTime'] = ap_datetime(poll.end_poll) + page['votersCount'] = 0 + choices = [] + for choice in PollChoice.query.filter_by(post_id=post.id).all(): + choices.append({'type': 'Note', 'name': choice.choice_text, 'replies': {'type': 'Collection', 'totalItems': 0}}) + page['oneOf' if poll.mode == 'single' else 'anyOf'] = choices + + activity = 'create' if not edit else 'update' + create_id = f"https://{current_app.config['SERVER_NAME']}/activities/{activity}/{gibberish(15)}" + type = 'Create' if not edit else 'Update' + create = { + 'id': create_id, + 'type': type, + 'actor': user.public_url(), + 'object': page, + 'to': to, + 'cc': cc, + '@context': default_context(), + 'tag': tag + } + + domains_sent_to = [current_app.config['SERVER_NAME']] + + # send the activity as an Announce if the community is local, or as a Create if not + if community.is_local(): + del create['@context'] + + announce_id = f"https://{current_app.config['SERVER_NAME']}/activities/announce/{gibberish(15)}" + actor = community.public_url() + cc = [community.ap_followers_url] + group_announce = { + 'id': announce_id, + 'type': 'Announce', + 'actor': community.public_url(), + 'object': create, + 'to': to, + 'cc': cc, + '@context': default_context() + } + microblog_announce = { + 'id': announce_id, + 'type': 'Announce', + 'actor': community.public_url(), + 'object': post.ap_id, + 'to': to, + 'cc': cc, + '@context': default_context() + } + for instance in community.following_instances(): + if instance.inbox and instance.online() and not user.has_blocked_instance(instance.id) and not instance_banned(instance.domain): + if instance.software in MICROBLOG_APPS: + post_request(instance.inbox, microblog_announce, community.private_key, community.public_url() + '#main-key') + else: + post_request(instance.inbox, group_announce, community.private_key, community.public_url() + '#main-key') + domains_sent_to.append(instance.domain) + else: + post_request(community.ap_inbox_url, create, user.private_key, user.public_url() + '#main-key') + domains_sent_to.append(community.instance.domain) + + # send copy of the Create to anyone else Mentioned in post, but not on an instance that's already sent to. + if '@context' not in create: + create['@context'] = default_context() + for recipient in recipients: + if recipient.instance.domain not in domains_sent_to: + post_request(recipient.instance.inbox, create, user.private_key, user.public_url() + '#main-key') + domains_sent_to.append(recipient.instance.domain) + + # send amended copy of the Create to anyone who is following the User, but hasn't already received something + followers = UserFollower.query.filter_by(local_user_id=post.user_id) + if not followers: + return + + if 'name' in page: + del page['name'] + note = page + note['type'] = 'Note' + if post.type == POST_TYPE_LINK or post.type == POST_TYPE_VIDEO: + note['content'] = '

' + post.title + '

' + else: + note['content'] = '

' + post.title + '

' + if post.body_html: + note['content'] = note['content'] + post.body_html + note['inReplyTo'] = None + create['object'] = note + + instances = Instance.query.join(User, User.instance_id == Instance.id).join(UserFollower, UserFollower.remote_user_id == User.id) + instances = instances.filter(UserFollower.local_user_id == post.user_id).filter(Instance.gone_forever == False) + for instance in instances: + if instance.domain not in domains_sent_to: + post_request(instance.inbox, create, user.private_key, user.public_url() + '#main-key') + From d64dde5468001e1bfb39e4cb1e9c9a664fd20c63 Mon Sep 17 00:00:00 2001 From: freamon Date: Wed, 15 Jan 2025 01:26:27 +0000 Subject: [PATCH 02/19] API: initial support for post edits (polls and uploaded images not supported yet) --- app/api/alpha/routes.py | 15 +- app/api/alpha/utils/__init__.py | 2 +- app/api/alpha/utils/post.py | 39 ++++- app/shared/post.py | 267 +++++++++++++++++++++++++++++++- 4 files changed, 313 insertions(+), 10 deletions(-) diff --git a/app/api/alpha/routes.py b/app/api/alpha/routes.py index 22eadfcf..5de30055 100644 --- a/app/api/alpha/routes.py +++ b/app/api/alpha/routes.py @@ -1,7 +1,7 @@ from app.api.alpha import bp from app.api.alpha.utils import get_site, post_site_block, \ get_search, \ - get_post_list, get_post, post_post_like, put_post_save, put_post_subscribe, post_post, \ + get_post_list, get_post, post_post_like, put_post_save, put_post_subscribe, post_post, put_post, \ get_reply_list, post_reply_like, put_reply_save, put_reply_subscribe, post_reply, put_reply, post_reply_delete, post_reply_report, \ get_community_list, get_community, post_community_follow, post_community_block, \ get_user, post_user_block @@ -171,6 +171,18 @@ def post_alpha_post(): return jsonify({"error": str(ex)}), 400 +@bp.route('/api/alpha/post', methods=['PUT']) +def put_alpha_post(): + if not enable_api(): + return jsonify({'error': 'alpha api is not enabled'}) + try: + auth = request.headers.get('Authorization') + data = request.get_json(force=True) or {} + return jsonify(put_post(auth, data)) + except Exception as ex: + return jsonify({"error": str(ex)}), 400 + + # Reply @bp.route('/api/alpha/comment/list', methods=['GET']) def get_alpha_comment_list(): @@ -333,7 +345,6 @@ def alpha_community(): return jsonify({"error": "not_yet_implemented"}), 400 # Post - not yet implemented -@bp.route('/api/alpha/post', methods=['PUT']) @bp.route('/api/alpha/post/delete', methods=['POST']) @bp.route('/api/alpha/post/remove', methods=['POST']) @bp.route('/api/alpha/post/lock', methods=['POST']) diff --git a/app/api/alpha/utils/__init__.py b/app/api/alpha/utils/__init__.py index e43dd700..a770a08c 100644 --- a/app/api/alpha/utils/__init__.py +++ b/app/api/alpha/utils/__init__.py @@ -1,6 +1,6 @@ from app.api.alpha.utils.site import get_site, post_site_block from app.api.alpha.utils.misc import get_search -from app.api.alpha.utils.post import get_post_list, get_post, post_post_like, put_post_save, put_post_subscribe, post_post +from app.api.alpha.utils.post import get_post_list, get_post, post_post_like, put_post_save, put_post_subscribe, post_post, put_post from app.api.alpha.utils.reply import get_reply_list, post_reply_like, put_reply_save, put_reply_subscribe, post_reply, put_reply, post_reply_delete, post_reply_report from app.api.alpha.utils.community import get_community, get_community_list, post_community_follow, post_community_block from app.api.alpha.utils.user import get_user, post_user_block diff --git a/app/api/alpha/utils/post.py b/app/api/alpha/utils/post.py index fd141057..a69f368e 100644 --- a/app/api/alpha/utils/post.py +++ b/app/api/alpha/utils/post.py @@ -1,9 +1,10 @@ from app import cache from app.api.alpha.views import post_view from app.api.alpha.utils.validators import required, integer_expected, boolean_expected, string_expected +from app.constants import POST_TYPE_ARTICLE, POST_TYPE_LINK, POST_TYPE_IMAGE, POST_TYPE_VIDEO from app.models import Post, Community, CommunityMember, utcnow -from app.shared.post import vote_for_post, bookmark_the_post, remove_the_bookmark_from_post, toggle_post_notification, make_post -from app.utils import authorise_api_user, blocked_users, blocked_communities, blocked_instances, community_ids_from_instances, is_image_url +from app.shared.post import vote_for_post, bookmark_the_post, remove_the_bookmark_from_post, toggle_post_notification, make_post, edit_post +from app.utils import authorise_api_user, blocked_users, blocked_communities, blocked_instances, community_ids_from_instances, is_image_url, is_video_url from datetime import timedelta from sqlalchemy import desc @@ -154,7 +155,7 @@ def post_post(auth, data): required(['title', 'community_id'], data) integer_expected(['language_id'], data) boolean_expected(['nsfw'], data) - string_expected(['string', 'body'], data) + string_expected(['title', 'body', 'url'], data) title = data['title'] community_id = data['community_id'] @@ -169,6 +170,8 @@ def post_post(auth, data): type = 'discussion' elif is_image_url(url): type = 'image' + elif is_video_url(url): + type = 'video' else: type = 'link' @@ -180,3 +183,33 @@ def post_post(auth, data): return post_json +def put_post(auth, data): + required(['post_id'], data) + integer_expected(['language_id'], data) + boolean_expected(['nsfw'], data) + string_expected(['title', 'body', 'url'], data) + + post_id = data['post_id'] + title = data['title'] + body = data['body'] if 'body' in data else '' + url = data['url'] if 'url' in data else None + nsfw = data['nsfw'] if 'nsfw' in data else False + language_id = data['language_id'] if 'language_id' in data else 2 # FIXME: use site language + if language_id < 2: + language_id = 2 + + if not url: + type = POST_TYPE_ARTICLE + elif is_image_url(url): + type = POST_TYPE_IMAGE + elif is_video_url(url): + type = POST_TYPE_VIDEO + else: + type = POST_TYPE_LINK + + input = {'title': title, 'body': body, 'url': url, 'nsfw': nsfw, 'language_id': language_id, 'notify_author': True} + post = Post.query.filter_by(id=post_id).one() + user_id, post = edit_post(input, post, type, SRC_API, auth) + + post_json = post_view(post=post, variant=4, user_id=user_id) + return post_json diff --git a/app/shared/post.py b/app/shared/post.py index 31e39d53..73430a6a 100644 --- a/app/shared/post.py +++ b/app/shared/post.py @@ -1,17 +1,21 @@ from app import db from app.constants import * -from app.community.util import tags_from_string -from app.models import Language, NotificationSubscription, Post, PostBookmark +from app.community.util import tags_from_string, tags_from_string_old, end_poll_date +from app.models import File, Language, NotificationSubscription, Poll, PollChoice, Post, PostBookmark, utcnow from app.shared.tasks import task_selector -from app.utils import render_template, authorise_api_user, shorten_string, gibberish, ensure_directory_exists +from app.utils import render_template, authorise_api_user, shorten_string, gibberish, ensure_directory_exists, \ + piefed_markdown_to_lemmy_markdown, markdown_to_html, remove_tracking_from_link, domain_from_url, \ + opengraph_parse, url_to_thumbnail_file, blocked_phrases -from flask import abort, flash, redirect, request, url_for, current_app +from flask import abort, flash, redirect, request, url_for, current_app, g from flask_babel import _ from flask_login import current_user from pillow_heif import register_heif_opener from PIL import Image, ImageOps +from sqlalchemy import text + import os # would be in app/constants.py @@ -240,3 +244,258 @@ def make_post(input, community, type, src, auth=None, uploaded_file=None): return post +def edit_post(input, post, type, src, auth=None, uploaded_file=None): + if src == SRC_API: + user = authorise_api_user(auth, return_type='model') + title = input['title'] + body = input['body'] + url = input['url'] + nsfw = input['nsfw'] + notify_author = input['notify_author'] + language_id = input['language_id'] + tags = [] + else: + user = current_user + title = input.title.data + body = input.body.data + url = input.link_url.data + nsfw = input.nsfw.data + notify_author = input.notify_author.data + language_id = input.language_id.data + tags = tags_from_string(input.tags.data) + + post.indexable = user.indexable + post.sticky = False if src == SRC_API else input.sticky.data + post.nsfw = nsfw + post.nsfl = False if src == SRC_API else input.nsfl.data + post.notify_author = notify_author + post.language_id = language_id + user.language_id = language_id + post.title = title.strip() + post.body = piefed_markdown_to_lemmy_markdown(body) + post.body_html = markdown_to_html(post.body) + + allowed_extensions = ['.gif', '.jpg', '.jpeg', '.png', '.webp', '.heic', '.mpo', '.avif', '.svg'] + + if not type or type == POST_TYPE_ARTICLE: + post.type = POST_TYPE_ARTICLE + elif type == POST_TYPE_LINK: + url_changed = post.id is None or url != post.url # post.id ? + post.url = remove_tracking_from_link(url.strip()) + post.type = POST_TYPE_LINK + domain = domain_from_url(url) + domain.post_count += 1 + post.domain = domain + + if url_changed: + if post.image_id: + remove_file = File.query.get(post.image_id) + remove_file.delete_from_disk() + post.image_id = None + + if post.url.endswith('.mp4') or post.url.endswith('.webm'): + post.type = POST_TYPE_VIDEO + file = File(source_url=url) # make_image_sizes() will take care of turning this into a still image + post.image = file + db.session.add(file) + else: + unused, file_extension = os.path.splitext(url) + # this url is a link to an image - turn it into a image post + if file_extension.lower() in allowed_extensions: + file = File(source_url=url) + post.image = file + db.session.add(file) + post.type = POST_TYPE_IMAGE + else: + # check opengraph tags on the page and make a thumbnail if an image is available in the og:image meta tag + if not post.type == POST_TYPE_VIDEO: + tn_url = url + if tn_url[:32] == 'https://www.youtube.com/watch?v=': + tn_url = 'https://youtu.be/' + tn_url[32:43] # better chance of thumbnail from youtu.be than youtube.com + opengraph = opengraph_parse(tn_url) + if opengraph and (opengraph.get('og:image', '') != '' or opengraph.get('og:image:url', '') != ''): + filename = opengraph.get('og:image') or opengraph.get('og:image:url') + if not filename.startswith('/'): + file = url_to_thumbnail_file(filename) + if file: + file.alt_text = shorten_string(opengraph.get('og:title'), 295) + post.image = file + db.session.add(file) + + elif type == POST_TYPE_IMAGE: + post.type = POST_TYPE_IMAGE + # If we are uploading new file in the place of existing one just remove the old one + if post.image_id is not None and src == SRC_WEB: + post.image.delete_from_disk() + image_id = post.image_id + post.image_id = None + db.session.add(post) + db.session.commit() + File.query.filter_by(id=image_id).delete() + + if uploaded_file and uploaded_file.filename != '' and src == SRC_WEB: + if post.image_id: + remove_file = File.query.get(post.image_id) + remove_file.delete_from_disk() + post.image_id = None + + # check if this is an allowed type of file + file_ext = os.path.splitext(uploaded_file.filename)[1] + if file_ext.lower() not in allowed_extensions: + abort(400) + new_filename = gibberish(15) + + # set up the storage directory + directory = 'app/static/media/posts/' + new_filename[0:2] + '/' + new_filename[2:4] + ensure_directory_exists(directory) + + # save the file + final_place = os.path.join(directory, new_filename + file_ext) + final_place_medium = os.path.join(directory, new_filename + '_medium.webp') + final_place_thumbnail = os.path.join(directory, new_filename + '_thumbnail.webp') + uploaded_file.seek(0) + uploaded_file.save(final_place) + + if file_ext.lower() == '.heic': + register_heif_opener() + + Image.MAX_IMAGE_PIXELS = 89478485 + + # resize if necessary + img = Image.open(final_place) + if '.' + img.format.lower() in allowed_extensions: + img = ImageOps.exif_transpose(img) + + # limit full sized version to 2000px + img_width = img.width + img_height = img.height + img.thumbnail((2000, 2000)) + img.save(final_place) + + # medium sized version + img.thumbnail((512, 512)) + img.save(final_place_medium, format="WebP", quality=93) + + # save a third, smaller, version as a thumbnail + img.thumbnail((170, 170)) + img.save(final_place_thumbnail, format="WebP", quality=93) + thumbnail_width = img.width + thumbnail_height = img.height + + alt_text = input.image_alt_text.data if input.image_alt_text.data else title + + file = File(file_path=final_place_medium, file_name=new_filename + file_ext, alt_text=alt_text, + width=img_width, height=img_height, thumbnail_width=thumbnail_width, + thumbnail_height=thumbnail_height, thumbnail_path=final_place_thumbnail, + source_url=final_place.replace('app/static/', f"https://{current_app.config['SERVER_NAME']}/static/")) + db.session.add(file) + db.session.commit() + post.image_id = file.id + + elif type == POST_TYPE_VIDEO: + url = url.strip() + url_changed = post.id is None or url != post.url + post.url = remove_tracking_from_link(url) + post.type = POST_TYPE_VIDEO + domain = domain_from_url(url) + domain.post_count += 1 + post.domain = domain + + if url_changed: + if post.image_id: + remove_file = File.query.get(post.image_id) + remove_file.delete_from_disk() + post.image_id = None + if url.endswith('.mp4') or url('.webm'): + file = File(source_url=url) # make_image_sizes() will take care of turning this into a still image + post.image = file + db.session.add(file) + else: + # check opengraph tags on the page and make a thumbnail if an image is available in the og:image meta tag + tn_url = url + if tn_url[:32] == 'https://www.youtube.com/watch?v=': + tn_url = 'https://youtu.be/' + tn_url[32:43] # better chance of thumbnail from youtu.be than youtube.com + opengraph = opengraph_parse(tn_url) + if opengraph and (opengraph.get('og:image', '') != '' or opengraph.get('og:image:url', '') != ''): + filename = opengraph.get('og:image') or opengraph.get('og:image:url') + if not filename.startswith('/'): + file = url_to_thumbnail_file(filename) + if file: + file.alt_text = shorten_string(opengraph.get('og:title'), 295) + post.image = file + db.session.add(file) + + elif type == POST_TYPE_POLL: + post.body = title + '\n' + body if title not in body else body + post.body_html = markdown_to_html(post.body) + post.type = POST_TYPE_POLL + else: + raise Exception('invalid post type') + + if post.id is None: + if user.reputation > 100: + post.up_votes = 1 + post.score = 1 + if user.reputation < -100: + post.score = -1 + post.ranking = post.post_ranking(post.score, utcnow()) + + # Filter by phrase + blocked_phrases_list = blocked_phrases() + for blocked_phrase in blocked_phrases_list: + if blocked_phrase in post.title: + abort(401) + return + if post.body: + for blocked_phrase in blocked_phrases_list: + if blocked_phrase in post.body: + abort(401) + return + + db.session.add(post) + else: + db.session.execute(text('DELETE FROM "post_tag" WHERE post_id = :post_id'), {'post_id': post.id}) + post.tags = tags + db.session.commit() + + # Save poll choices. NB this will delete all votes whenever a poll is edited. Partially because it's easier to code but also to stop malicious alterations to polls after people have already voted + if type == POST_TYPE_POLL: + db.session.execute(text('DELETE FROM "poll_choice_vote" WHERE post_id = :post_id'), {'post_id': post.id}) + db.session.execute(text('DELETE FROM "poll_choice" WHERE post_id = :post_id'), {'post_id': post.id}) + for i in range(1, 10): + choice_data = getattr(input, f"choice_{i}").data.strip() + if choice_data != '': + db.session.add(PollChoice(post_id=post.id, choice_text=choice_data, sort_order=i)) + + poll = Poll.query.filter_by(post_id=post.id).first() + if poll is None: + poll = Poll(post_id=post.id) + db.session.add(poll) + poll.mode = input.mode.data + if input.finish_in: + poll.end_poll = end_poll_date(input.finish_in.data) + poll.local_only = input.local_only.data + poll.latest_vote = None + db.session.commit() + + # Notify author about replies + # Remove any subscription that currently exists + existing_notification = NotificationSubscription.query.filter(NotificationSubscription.entity_id == post.id, + NotificationSubscription.user_id == user.id, + NotificationSubscription.type == NOTIF_POST).first() + if existing_notification: + db.session.delete(existing_notification) + + # Add subscription if necessary + if notify_author: + new_notification = NotificationSubscription(name=post.title, user_id=user.id, entity_id=post.id, + type=NOTIF_POST) + db.session.add(new_notification) + + g.site.last_active = utcnow() + db.session.commit() + + task_selector('edit_post', user_id=user.id, post_id=post.id) + + if src == SRC_API: + return user.id, post From e85a0ddd9ae92079bced10c555e1ef553723190f Mon Sep 17 00:00:00 2001 From: freamon Date: Thu, 16 Jan 2025 14:40:02 +0000 Subject: [PATCH 03/19] Keep post.cross_posts at null instead of empty array if they don't have any --- app/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models.py b/app/models.py index d83cb360..5c14121a 100644 --- a/app/models.py +++ b/app/models.py @@ -1457,7 +1457,7 @@ class Post(db.Model): ncp.cross_posts.append(self.id) # this post: set the cross_posts field to the limited list of ids from the most recent other posts - if new_cross_posts: + if new_cross_posts.count() > 0: self.cross_posts = [ncp.id for ncp in new_cross_posts] db.session.commit() From f9c999137021f24ccf1c6061c15bdcce7eb84b65 Mon Sep 17 00:00:00 2001 From: freamon Date: Fri, 17 Jan 2025 08:17:14 +0000 Subject: [PATCH 04/19] bugfix for blocked phrases --- app/utils.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/app/utils.py b/app/utils.py index 28ee7b6b..328d382b 100644 --- a/app/utils.py +++ b/app/utils.py @@ -557,7 +557,14 @@ def blocked_users(user_id) -> List[int]: def blocked_phrases() -> List[str]: site = Site.query.get(1) if site.blocked_phrases: - return [phrase for phrase in site.blocked_phrases.split('\n') if phrase != ''] + blocked_phrases = [] + for phrase in site.blocked_phrases.split('\n'): + if phrase != '': + if phrase.endswith('\r'): + blocked_phrases.append(phrase[:-1]) + else: + blocked_phrases.append(phrase) + return blocked_phrases else: return [] From 989afa0344a8f39835ff8b4cda883377fb9d3fac Mon Sep 17 00:00:00 2001 From: freamon Date: Fri, 17 Jan 2025 17:18:38 +0000 Subject: [PATCH 05/19] fixes for outgoing federation of comments --- app/shared/tasks/notes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/shared/tasks/notes.py b/app/shared/tasks/notes.py index 5f424621..1d38aa68 100644 --- a/app/shared/tasks/notes.py +++ b/app/shared/tasks/notes.py @@ -149,7 +149,7 @@ def send_reply(user_id, reply_id, parent_id, edit=False): 'distinguished': False, } if edit: - note['updated']: ap_datetime(utcnow()) + note['updated'] = ap_datetime(utcnow()) activity = 'create' if not edit else 'update' create_id = f"https://{current_app.config['SERVER_NAME']}/activities/{activity}/{gibberish(15)}" @@ -162,6 +162,7 @@ def send_reply(user_id, reply_id, parent_id, edit=False): 'to': to, 'cc': cc, '@context': default_context(), + 'audience': community.public_url(), 'tag': tag } From 19771a5ea4584e32ab149b37e5bfcc47808bb122 Mon Sep 17 00:00:00 2001 From: freamon Date: Fri, 17 Jan 2025 17:20:15 +0000 Subject: [PATCH 06/19] API: merge up make_post and edit_post --- app/api/alpha/utils/post.py | 20 +- app/shared/post.py | 487 +++++++++++++++--------------------- app/shared/tasks/pages.py | 116 +++++---- 3 files changed, 272 insertions(+), 351 deletions(-) diff --git a/app/api/alpha/utils/post.py b/app/api/alpha/utils/post.py index a69f368e..12510c8d 100644 --- a/app/api/alpha/utils/post.py +++ b/app/api/alpha/utils/post.py @@ -166,14 +166,8 @@ def post_post(auth, data): if language_id < 2: language_id = 2 - if not url: - type = 'discussion' - elif is_image_url(url): - type = 'image' - elif is_video_url(url): - type = 'video' - else: - type = 'link' + # change when Polls are supported + type = None input = {'title': title, 'body': body, 'url': url, 'nsfw': nsfw, 'language_id': language_id, 'notify_author': True} community = Community.query.filter_by(id=community_id).one() @@ -198,14 +192,8 @@ def put_post(auth, data): if language_id < 2: language_id = 2 - if not url: - type = POST_TYPE_ARTICLE - elif is_image_url(url): - type = POST_TYPE_IMAGE - elif is_video_url(url): - type = POST_TYPE_VIDEO - else: - type = POST_TYPE_LINK + # change when Polls are supported + type = None input = {'title': title, 'body': body, 'url': url, 'nsfw': nsfw, 'language_id': language_id, 'notify_author': True} post = Post.query.filter_by(id=post_id).one() diff --git a/app/shared/post.py b/app/shared/post.py index 73430a6a..7804fb12 100644 --- a/app/shared/post.py +++ b/app/shared/post.py @@ -1,11 +1,13 @@ -from app import db +from app import db, cache +from app.activitypub.util import make_image_sizes from app.constants import * -from app.community.util import tags_from_string, tags_from_string_old, end_poll_date -from app.models import File, Language, NotificationSubscription, Poll, PollChoice, Post, PostBookmark, utcnow +from app.community.util import tags_from_string_old, end_poll_date +from app.models import File, Language, NotificationSubscription, Poll, PollChoice, Post, PostBookmark, PostVote, utcnow from app.shared.tasks import task_selector from app.utils import render_template, authorise_api_user, shorten_string, gibberish, ensure_directory_exists, \ piefed_markdown_to_lemmy_markdown, markdown_to_html, remove_tracking_from_link, domain_from_url, \ - opengraph_parse, url_to_thumbnail_file, blocked_phrases + opengraph_parse, url_to_thumbnail_file, can_create_post, is_video_hosting_site, recently_upvoted_posts, \ + is_image_url, is_video_hosting_site from flask import abort, flash, redirect, request, url_for, current_app, g from flask_babel import _ @@ -144,6 +146,76 @@ def make_post(input, community, type, src, auth=None, uploaded_file=None): #if not basic_rate_limit_check(user): # raise Exception('rate_limited') title = input['title'] + url = input['url'] + language_id = input['language_id'] + else: + user = current_user + title = input.title.data.strip() + if type == POST_TYPE_LINK: + url = input.link_url.data.strip() + elif type == POST_TYPE_VIDEO: + url = input.video_url.data.strip() + else: + url = None + language_id = input.language_id.data + + # taking values from one JSON to put in another JSON to put in a DB to put in another JSON feels bad + # instead, make_post shares code with edit_post + # ideally, a similar change could be made for incoming activitypub (create_post() and update_post_from_activity() could share code) + # once this happens, and post.new() just does the minimum before being passed off to an update function, post.new() can be used here again. + + if not can_create_post(user, community): + raise Exception('You are not permitted to make posts in this community') + + if url: + domain = domain_from_url(url) + if domain: + if domain.banned or domain.name.endswith('.pages.dev'): + raise Exception(domain.name + ' is blocked by admin') + + if uploaded_file and uploaded_file.filename != '': + # check if this is an allowed type of file + allowed_extensions = ['.gif', '.jpg', '.jpeg', '.png', '.webp', '.heic', '.mpo', '.avif', '.svg'] + file_ext = os.path.splitext(uploaded_file.filename)[1] + if file_ext.lower() not in allowed_extensions: + raise Exception('filetype not allowed') + + post = Post(user_id=user.id, community_id=community.id, instance_id=user.instance_id, posted_at=utcnow(), + ap_id=gibberish(), title=title, language_id=language_id) + db.session.add(post) + db.session.commit() + + post.up_votes = 1 + if user.reputation > 100: + post.up_votes += 1 + effect = user.instance.vote_weight + post.score = post.up_votes * effect + post.ranking = post.post_ranking(post.score, post.posted_at) + cache.delete_memoized(recently_upvoted_posts, user.id) + + community.post_count += 1 + community.last_active = g.site.last_active = utcnow() + user.post_count += 1 + + post.ap_id = f"https://{current_app.config['SERVER_NAME']}/post/{post.id}" + vote = PostVote(user_id=user.id, post_id=post.id, author_id=user.id, effect=1) + db.session.add(vote) + db.session.commit() + + post = edit_post(input, post, type, src, user, auth, uploaded_file, from_scratch=True) + + if src == SRC_API: + return user.id, post + else: + return post + + +# 'from_scratch == True' means that it's not really a user edit, we're just re-using code for make_post() +def edit_post(input, post, type, src, user=None, auth=None, uploaded_file=None, from_scratch=False): + if src == SRC_API: + if not user: + user = authorise_api_user(auth, return_type='model') + title = input['title'] body = input['body'] url = input['url'] nsfw = input['nsfw'] @@ -151,46 +223,82 @@ def make_post(input, community, type, src, auth=None, uploaded_file=None): language_id = input['language_id'] tags = [] else: - user = current_user - title = input.title.data + if not user: + user = current_user + title = input.title.data.strip() body = input.body.data - url = input.link_url.data + if type == POST_TYPE_LINK: + url = input.link_url.data.strip() + elif type == POST_TYPE_VIDEO: + url = input.video_url.data.strip() + elif type == POST_TYPE_IMAGE and not from_scratch: + url = post.url + else: + url = None nsfw = input.nsfw.data notify_author = input.notify_author.data language_id = input.language_id.data - tags = tags_from_string(input.tags.data) + tags = tags_from_string_old(input.tags.data) - language = Language.query.filter_by(id=language_id).one() + post.indexable = user.indexable + post.sticky = False if src == SRC_API else input.sticky.data + post.nsfw = nsfw + post.nsfl = False if src == SRC_API else input.nsfl.data + post.notify_author = notify_author + post.language_id = language_id + user.language_id = language_id + post.title = title + post.body = piefed_markdown_to_lemmy_markdown(body) + post.body_html = markdown_to_html(post.body) + post.type = type - request_json = { - 'id': None, - 'object': { - 'name': title, - 'type': 'Page', - 'stickied': False if src == SRC_API else input.sticky.data, - 'sensitive': nsfw, - 'nsfl': False if src == SRC_API else input.nsfl.data, - 'id': gibberish(), # this will be updated once we have the post.id - 'mediaType': 'text/markdown', - 'content': body, - 'tag': tags, - 'language': {'identifier': language.code, 'name': language.name} - } - } + url_changed = False - if type == 'link' or (type == 'image' and src == SRC_API): - request_json['object']['attachment'] = [{'type': 'Link', 'href': url}] - elif type == 'image' and src == SRC_WEB and uploaded_file and uploaded_file.filename != '': + if not from_scratch: + # Remove any subscription that currently exists + existing_notification = NotificationSubscription.query.filter(NotificationSubscription.entity_id == post.id, + NotificationSubscription.user_id == user.id, + NotificationSubscription.type == NOTIF_POST).first() + if existing_notification: + db.session.delete(existing_notification) + + # Remove any poll votes that currently exists + # Partially because it's easier to code but also to stop malicious alterations to polls after people have already voted + db.session.execute(text('DELETE FROM "poll_choice_vote" WHERE post_id = :post_id'), {'post_id': post.id}) + db.session.execute(text('DELETE FROM "poll_choice" WHERE post_id = :post_id'), {'post_id': post.id}) + + # Remove any old images, and set url_changed + if url != post.url or uploaded_file: + url_changed = True + if post.image_id: + remove_file = File.query.get(post.image_id) + if remove_file: + remove_file.delete_from_disk() + post.image_id = None + domain = domain_from_url(post.url) + if domain: + domain.post_count -= 1 + + # remove any old tags + db.session.execute(text('DELETE FROM "post_tag" WHERE post_id = :post_id'), {'post_id': post.id}) + + post.edited_at = utcnow() + + db.session.commit() + + if uploaded_file and uploaded_file.filename != '': # check if this is an allowed type of file + allowed_extensions = ['.gif', '.jpg', '.jpeg', '.png', '.webp', '.heic', '.mpo', '.avif', '.svg'] file_ext = os.path.splitext(uploaded_file.filename)[1] if file_ext.lower() not in allowed_extensions: - abort(400, description="Invalid image type.") + raise Exception('filetype not allowed') new_filename = gibberish(15) # set up the storage directory directory = 'app/static/media/posts/' + new_filename[0:2] + '/' + new_filename[2:4] ensure_directory_exists(directory) + # save the file final_place = os.path.join(directory, new_filename + file_ext) uploaded_file.seek(0) uploaded_file.save(final_place) @@ -202,267 +310,74 @@ def make_post(input, community, type, src, auth=None, uploaded_file=None): Image.MAX_IMAGE_PIXELS = 89478485 - # resize if necessary + # limit full sized version to 2000px if not final_place.endswith('.svg'): img = Image.open(final_place) if '.' + img.format.lower() in allowed_extensions: img = ImageOps.exif_transpose(img) - # limit full sized version to 2000px img.thumbnail((2000, 2000)) img.save(final_place) - request_json['object']['attachment'] = [{ - 'type': 'Image', - 'url': f'https://{current_app.config["SERVER_NAME"]}/{final_place.replace("app/", "")}', - 'name': input.image_alt_text.data, - 'file_path': final_place - }] - elif type == 'video': - request_json['object']['attachment'] = [{'type': 'Document', 'url': url}] - elif type == 'poll': - request_json['object']['type'] = 'Question' - choices = [input.choice_1, input.choice_2, input.choice_3, input.choice_4, input.choice_5, - input.choice_6, input.choice_7, input.choice_8, input.choice_9, input.choice_10] - key = 'oneOf' if input.mode.data == 'single' else 'anyOf' - request_json['object'][key] = [] - for choice in choices: - choice_data = choice.data.strip() - if choice_data: - request_json['object'][key].append({'name': choice_data}) - request_json['object']['endTime'] = end_poll_date(input.finish_in.data) - - post = Post.new(user, community, request_json) - post.ap_id = f"https://{current_app.config['SERVER_NAME']}/post/{post.id}" - db.session.commit() - - task_selector('make_post', user_id=user.id, post_id=post.id) - - if src == SRC_API: - return user.id, post - else: - return post - - -def edit_post(input, post, type, src, auth=None, uploaded_file=None): - if src == SRC_API: - user = authorise_api_user(auth, return_type='model') - title = input['title'] - body = input['body'] - url = input['url'] - nsfw = input['nsfw'] - notify_author = input['notify_author'] - language_id = input['language_id'] - tags = [] - else: - user = current_user - title = input.title.data - body = input.body.data - url = input.link_url.data - nsfw = input.nsfw.data - notify_author = input.notify_author.data - language_id = input.language_id.data - tags = tags_from_string(input.tags.data) - - post.indexable = user.indexable - post.sticky = False if src == SRC_API else input.sticky.data - post.nsfw = nsfw - post.nsfl = False if src == SRC_API else input.nsfl.data - post.notify_author = notify_author - post.language_id = language_id - user.language_id = language_id - post.title = title.strip() - post.body = piefed_markdown_to_lemmy_markdown(body) - post.body_html = markdown_to_html(post.body) - - allowed_extensions = ['.gif', '.jpg', '.jpeg', '.png', '.webp', '.heic', '.mpo', '.avif', '.svg'] - - if not type or type == POST_TYPE_ARTICLE: - post.type = POST_TYPE_ARTICLE - elif type == POST_TYPE_LINK: - url_changed = post.id is None or url != post.url # post.id ? - post.url = remove_tracking_from_link(url.strip()) - post.type = POST_TYPE_LINK - domain = domain_from_url(url) - domain.post_count += 1 - post.domain = domain - - if url_changed: - if post.image_id: - remove_file = File.query.get(post.image_id) - remove_file.delete_from_disk() - post.image_id = None - - if post.url.endswith('.mp4') or post.url.endswith('.webm'): - post.type = POST_TYPE_VIDEO - file = File(source_url=url) # make_image_sizes() will take care of turning this into a still image - post.image = file - db.session.add(file) + url = f"https://{current_app.config['SERVER_NAME']}/{final_place.replace('app/', '')}" else: - unused, file_extension = os.path.splitext(url) - # this url is a link to an image - turn it into a image post - if file_extension.lower() in allowed_extensions: - file = File(source_url=url) - post.image = file - db.session.add(file) - post.type = POST_TYPE_IMAGE - else: - # check opengraph tags on the page and make a thumbnail if an image is available in the og:image meta tag - if not post.type == POST_TYPE_VIDEO: - tn_url = url - if tn_url[:32] == 'https://www.youtube.com/watch?v=': - tn_url = 'https://youtu.be/' + tn_url[32:43] # better chance of thumbnail from youtu.be than youtube.com - opengraph = opengraph_parse(tn_url) - if opengraph and (opengraph.get('og:image', '') != '' or opengraph.get('og:image:url', '') != ''): - filename = opengraph.get('og:image') or opengraph.get('og:image:url') - if not filename.startswith('/'): - file = url_to_thumbnail_file(filename) - if file: - file.alt_text = shorten_string(opengraph.get('og:title'), 295) - post.image = file - db.session.add(file) + raise Exception('filetype not allowed') - elif type == POST_TYPE_IMAGE: - post.type = POST_TYPE_IMAGE - # If we are uploading new file in the place of existing one just remove the old one - if post.image_id is not None and src == SRC_WEB: - post.image.delete_from_disk() - image_id = post.image_id - post.image_id = None - db.session.add(post) + if url and (from_scratch or url_changed): + domain = domain_from_url(url) + if domain: + if domain.banned or domain.name.endswith('.pages.dev'): + raise Exception(domain.name + ' is blocked by admin') + post.domain = domain + domain.post_count += 1 + already_notified = set() # often admins and mods are the same people - avoid notifying them twice + if domain.notify_mods: + for community_member in post.community.moderators(): + if community_member.is_local(): + notify = Notification(title='Suspicious content', url=post.ap_id, user_id=community_member.user_id, author_id=user.id) + db.session.add(notify) + already_notified.add(community_member.user_id) + if domain.notify_admins: + for admin in Site.admins(): + if admin.id not in already_notified: + notify = Notification(title='Suspicious content', url=post.ap_id, user_id=admin.id, author_id=user.id) + db.session.add(notify) + if is_image_url(url): + file = File(source_url=url) + if uploaded_file and type == POST_TYPE_IMAGE: + # change this line when uploaded_file is supported in API + file.alt_text = input.image_alt_text.data if input.image_alt_text.data else title + db.session.add(file) db.session.commit() - File.query.filter_by(id=image_id).delete() + post.image_id = file.id + make_image_sizes(post.image_id, 170, 512, 'posts', post.community.low_quality) + post.url = url + post.type = POST_TYPE_IMAGE + else: + opengraph = opengraph_parse(url) + if opengraph and (opengraph.get('og:image', '') != '' or opengraph.get('og:image:url', '') != ''): + filename = opengraph.get('og:image') or opengraph.get('og:image:url') + if not filename.startswith('/'): + file = url_to_thumbnail_file(filename) + if file: + file.alt_text = shorten_string(opengraph.get('og:title'), 295) + post.image = file + db.session.add(file) - if uploaded_file and uploaded_file.filename != '' and src == SRC_WEB: - if post.image_id: - remove_file = File.query.get(post.image_id) - remove_file.delete_from_disk() - post.image_id = None + post.url = remove_tracking_from_link(url) - # check if this is an allowed type of file - file_ext = os.path.splitext(uploaded_file.filename)[1] - if file_ext.lower() not in allowed_extensions: - abort(400) - new_filename = gibberish(15) - - # set up the storage directory - directory = 'app/static/media/posts/' + new_filename[0:2] + '/' + new_filename[2:4] - ensure_directory_exists(directory) - - # save the file - final_place = os.path.join(directory, new_filename + file_ext) - final_place_medium = os.path.join(directory, new_filename + '_medium.webp') - final_place_thumbnail = os.path.join(directory, new_filename + '_thumbnail.webp') - uploaded_file.seek(0) - uploaded_file.save(final_place) - - if file_ext.lower() == '.heic': - register_heif_opener() - - Image.MAX_IMAGE_PIXELS = 89478485 - - # resize if necessary - img = Image.open(final_place) - if '.' + img.format.lower() in allowed_extensions: - img = ImageOps.exif_transpose(img) - - # limit full sized version to 2000px - img_width = img.width - img_height = img.height - img.thumbnail((2000, 2000)) - img.save(final_place) - - # medium sized version - img.thumbnail((512, 512)) - img.save(final_place_medium, format="WebP", quality=93) - - # save a third, smaller, version as a thumbnail - img.thumbnail((170, 170)) - img.save(final_place_thumbnail, format="WebP", quality=93) - thumbnail_width = img.width - thumbnail_height = img.height - - alt_text = input.image_alt_text.data if input.image_alt_text.data else title - - file = File(file_path=final_place_medium, file_name=new_filename + file_ext, alt_text=alt_text, - width=img_width, height=img_height, thumbnail_width=thumbnail_width, - thumbnail_height=thumbnail_height, thumbnail_path=final_place_thumbnail, - source_url=final_place.replace('app/static/', f"https://{current_app.config['SERVER_NAME']}/static/")) - db.session.add(file) - db.session.commit() - post.image_id = file.id - - elif type == POST_TYPE_VIDEO: - url = url.strip() - url_changed = post.id is None or url != post.url - post.url = remove_tracking_from_link(url) - post.type = POST_TYPE_VIDEO - domain = domain_from_url(url) - domain.post_count += 1 - post.domain = domain - - if url_changed: - if post.image_id: - remove_file = File.query.get(post.image_id) - remove_file.delete_from_disk() - post.image_id = None - if url.endswith('.mp4') or url('.webm'): - file = File(source_url=url) # make_image_sizes() will take care of turning this into a still image - post.image = file - db.session.add(file) + if url.endswith('.mp4') or url.endswith('.webm') or is_video_hosting_site(url): + post.type = POST_TYPE_VIDEO else: - # check opengraph tags on the page and make a thumbnail if an image is available in the og:image meta tag - tn_url = url - if tn_url[:32] == 'https://www.youtube.com/watch?v=': - tn_url = 'https://youtu.be/' + tn_url[32:43] # better chance of thumbnail from youtu.be than youtube.com - opengraph = opengraph_parse(tn_url) - if opengraph and (opengraph.get('og:image', '') != '' or opengraph.get('og:image:url', '') != ''): - filename = opengraph.get('og:image') or opengraph.get('og:image:url') - if not filename.startswith('/'): - file = url_to_thumbnail_file(filename) - if file: - file.alt_text = shorten_string(opengraph.get('og:title'), 295) - post.image = file - db.session.add(file) + post.type = POST_TYPE_LINK - elif type == POST_TYPE_POLL: - post.body = title + '\n' + body if title not in body else body - post.body_html = markdown_to_html(post.body) - post.type = POST_TYPE_POLL - else: - raise Exception('invalid post type') + post.calculate_cross_posts(url_changed=url_changed) - if post.id is None: - if user.reputation > 100: - post.up_votes = 1 - post.score = 1 - if user.reputation < -100: - post.score = -1 - post.ranking = post.post_ranking(post.score, utcnow()) - - # Filter by phrase - blocked_phrases_list = blocked_phrases() - for blocked_phrase in blocked_phrases_list: - if blocked_phrase in post.title: - abort(401) - return - if post.body: - for blocked_phrase in blocked_phrases_list: - if blocked_phrase in post.body: - abort(401) - return - - db.session.add(post) - else: - db.session.execute(text('DELETE FROM "post_tag" WHERE post_id = :post_id'), {'post_id': post.id}) - post.tags = tags - db.session.commit() - - # Save poll choices. NB this will delete all votes whenever a poll is edited. Partially because it's easier to code but also to stop malicious alterations to polls after people have already voted + federate = True if type == POST_TYPE_POLL: - db.session.execute(text('DELETE FROM "poll_choice_vote" WHERE post_id = :post_id'), {'post_id': post.id}) - db.session.execute(text('DELETE FROM "poll_choice" WHERE post_id = :post_id'), {'post_id': post.id}) + post.type = POST_TYPE_POLL for i in range(1, 10): + # change this line when polls are supported in API choice_data = getattr(input, f"choice_{i}").data.strip() if choice_data != '': db.session.add(PollChoice(post_id=post.id, choice_text=choice_data, sort_order=i)) @@ -478,24 +393,30 @@ def edit_post(input, post, type, src, auth=None, uploaded_file=None): poll.latest_vote = None db.session.commit() - # Notify author about replies - # Remove any subscription that currently exists - existing_notification = NotificationSubscription.query.filter(NotificationSubscription.entity_id == post.id, - NotificationSubscription.user_id == user.id, - NotificationSubscription.type == NOTIF_POST).first() - if existing_notification: - db.session.delete(existing_notification) + if poll.local_only: + federate = False + + # add tags + post.tags = tags # Add subscription if necessary if notify_author: - new_notification = NotificationSubscription(name=post.title, user_id=user.id, entity_id=post.id, - type=NOTIF_POST) + new_notification = NotificationSubscription(name=post.title, user_id=user.id, entity_id=post.id, type=NOTIF_POST) db.session.add(new_notification) - g.site.last_active = utcnow() db.session.commit() - task_selector('edit_post', user_id=user.id, post_id=post.id) + if from_scratch: + if federate: + task_selector('make_post', user_id=user.id, post_id=post.id) + elif federate: + task_selector('edit_post', user_id=user.id, post_id=post.id) if src == SRC_API: - return user.id, post + if from_scratch: + return post + else: + return user.id, post + elif from_scratch: + return post + diff --git a/app/shared/tasks/pages.py b/app/shared/tasks/pages.py index 0499d59e..e88701db 100644 --- a/app/shared/tasks/pages.py +++ b/app/shared/tasks/pages.py @@ -1,7 +1,7 @@ from app import celery, db from app.activitypub.signature import default_context, post_request from app.constants import POST_TYPE_LINK, POST_TYPE_ARTICLE, POST_TYPE_IMAGE, POST_TYPE_VIDEO, POST_TYPE_POLL, MICROBLOG_APPS -from app.models import CommunityBan, Instance, Notification, Post, User, UserFollower, utcnow +from app.models import CommunityBan, Instance, Notification, Poll, PollChoice, Post, User, UserFollower, utcnow from app.user.utils import search_for_user from app.utils import gibberish, instance_banned, ap_datetime @@ -113,7 +113,13 @@ def send_post(user_id, post_id, edit=False): db.session.add(notification) db.session.commit() - if community.local_only or not community.instance.online(): + if not community.instance.online(): + return + + # local_only communities can also be used to send activity to User Followers + # return now though, if there aren't any + followers = UserFollower.query.filter_by(local_user_id=post.user_id).first() + if not followers and community.local_only: return banned = CommunityBan.query.filter_by(user_id=user_id, community_id=community.id).first() @@ -146,7 +152,7 @@ def send_post(user_id, post_id, edit=False): 'cc': cc, 'tag': tag, 'audience': community.public_url(), - 'content': post.body_html if post.type != POST_TYPE_POLL else '

' + post.title + '

', + 'content': post.body_html if post.type != POST_TYPE_POLL else '

' + post.title + '

' + post.body_html, 'mediaType': 'text/html', 'source': source, 'published': ap_datetime(post.posted_at), @@ -161,7 +167,7 @@ def send_post(user_id, post_id, edit=False): if post.type != POST_TYPE_POLL: page['name'] = post.title if edit: - page['updated']: ap_datetime(utcnow()) + page['updated'] = ap_datetime(utcnow()) if post.image_id: image_url = '' if post.image.source_url: @@ -174,10 +180,10 @@ def send_post(user_id, post_id, edit=False): if post.type == POST_TYPE_POLL: poll = Poll.query.filter_by(post_id=post.id).first() page['endTime'] = ap_datetime(poll.end_poll) - page['votersCount'] = 0 + page['votersCount'] = poll.total_votes() if edit else 0 choices = [] for choice in PollChoice.query.filter_by(post_id=post.id).all(): - choices.append({'type': 'Note', 'name': choice.choice_text, 'replies': {'type': 'Collection', 'totalItems': 0}}) + choices.append({'type': 'Note', 'name': choice.choice_text, 'replies': {'type': 'Collection', 'totalItems': choice.num_votes if edit else 0}}) page['oneOf' if poll.mode == 'single' else 'anyOf'] = choices activity = 'create' if not edit else 'update' @@ -191,64 +197,70 @@ def send_post(user_id, post_id, edit=False): 'to': to, 'cc': cc, '@context': default_context(), + 'audience': community.public_url(), 'tag': tag } domains_sent_to = [current_app.config['SERVER_NAME']] - # send the activity as an Announce if the community is local, or as a Create if not - if community.is_local(): - del create['@context'] + # if the community is local, and remote instance is something like Lemmy, Announce the activity + # if the community is local, and remote instance is something like Mastodon, Announce creates (so the community Boosts it), but send updates directly and from the user + # Announce of Poll doesn't work for Mastodon, so don't add domain to domains_sent_to, so they receive it if they're also following the User or they get Mentioned + # if the community is remote, send activity directly + if not community.local_only: + if community.is_local(): + del create['@context'] - announce_id = f"https://{current_app.config['SERVER_NAME']}/activities/announce/{gibberish(15)}" - actor = community.public_url() - cc = [community.ap_followers_url] - group_announce = { - 'id': announce_id, - 'type': 'Announce', - 'actor': community.public_url(), - 'object': create, - 'to': to, - 'cc': cc, - '@context': default_context() - } - microblog_announce = { - 'id': announce_id, - 'type': 'Announce', - 'actor': community.public_url(), - 'object': post.ap_id, - 'to': to, - 'cc': cc, - '@context': default_context() - } - for instance in community.following_instances(): - if instance.inbox and instance.online() and not user.has_blocked_instance(instance.id) and not instance_banned(instance.domain): - if instance.software in MICROBLOG_APPS: - post_request(instance.inbox, microblog_announce, community.private_key, community.public_url() + '#main-key') - else: - post_request(instance.inbox, group_announce, community.private_key, community.public_url() + '#main-key') - domains_sent_to.append(instance.domain) - else: - post_request(community.ap_inbox_url, create, user.private_key, user.public_url() + '#main-key') - domains_sent_to.append(community.instance.domain) + announce_id = f"https://{current_app.config['SERVER_NAME']}/activities/announce/{gibberish(15)}" + actor = community.public_url() + cc = [community.ap_followers_url] + group_announce = { + 'id': announce_id, + 'type': 'Announce', + 'actor': community.public_url(), + 'object': create, + 'to': to, + 'cc': cc, + '@context': default_context() + } + microblog_announce = { + 'id': announce_id, + 'type': 'Announce', + 'actor': community.public_url(), + 'object': post.ap_id, + 'to': to, + 'cc': cc, + '@context': default_context() + } + for instance in community.following_instances(): + if instance.inbox and instance.online() and not user.has_blocked_instance(instance.id) and not instance_banned(instance.domain): + if instance.software in MICROBLOG_APPS: + if activity == 'create': + post_request(instance.inbox, microblog_announce, community.private_key, community.public_url() + '#main-key') + else: + post_request(instance.inbox, create, user.private_key, user.public_url() + '#main-key') + else: + post_request(instance.inbox, group_announce, community.private_key, community.public_url() + '#main-key') + if post.type < POST_TYPE_POLL: + domains_sent_to.append(instance.domain) + else: + post_request(community.ap_inbox_url, create, user.private_key, user.public_url() + '#main-key') + domains_sent_to.append(community.instance.domain) - # send copy of the Create to anyone else Mentioned in post, but not on an instance that's already sent to. - if '@context' not in create: - create['@context'] = default_context() - for recipient in recipients: - if recipient.instance.domain not in domains_sent_to: - post_request(recipient.instance.inbox, create, user.private_key, user.public_url() + '#main-key') - domains_sent_to.append(recipient.instance.domain) + # send copy of the Create to anyone else Mentioned in post, but not on an instance that's already sent to. + if '@context' not in create: + create['@context'] = default_context() + for recipient in recipients: + if recipient.instance.domain not in domains_sent_to: + post_request(recipient.instance.inbox, create, user.private_key, user.public_url() + '#main-key') + domains_sent_to.append(recipient.instance.domain) # send amended copy of the Create to anyone who is following the User, but hasn't already received something - followers = UserFollower.query.filter_by(local_user_id=post.user_id) - if not followers: - return - if 'name' in page: del page['name'] note = page - note['type'] = 'Note' + if note['type'] == 'Page': + note['type'] = 'Note' if post.type == POST_TYPE_LINK or post.type == POST_TYPE_VIDEO: note['content'] = '

' + post.title + '

' else: From 6ee41578d6bd1a226921b48e9ac9e16d6d25717c Mon Sep 17 00:00:00 2001 From: freamon Date: Fri, 17 Jan 2025 17:24:26 +0000 Subject: [PATCH 07/19] mentions in post bodies - use shared code for outbound federation / notification --- app/activitypub/util.py | 14 -- app/community/routes.py | 300 +--------------------------------------- app/community/util.py | 227 ------------------------------ app/post/routes.py | 216 ++--------------------------- 4 files changed, 21 insertions(+), 736 deletions(-) diff --git a/app/activitypub/util.py b/app/activitypub/util.py index fd94c34b..898a509c 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -2569,21 +2569,7 @@ def verify_object_from_source(request_json): return request_json -# This is for followers on microblog apps -# Used to let them know a Poll has been updated with a new vote -# The plan is to also use it for activities on local user's posts that aren't understood by being Announced (anything beyond the initial Create) -# This would need for posts to have things like a 'Replies' collection and a 'Likes' collection, so these can be downloaded when the post updates -# Using collecions like this (as PeerTube does) circumvents the problem of not having a remote user's private key. -# The problem of what to do for remote user's activity on a remote user's post in a local community still exists (can't Announce it, can't inform of post update) def inform_followers_of_post_update(post_id: int, sending_instance_id: int): - if current_app.debug: - inform_followers_of_post_update_task(post_id, sending_instance_id) - else: - inform_followers_of_post_update_task.delay(post_id, sending_instance_id) - - -@celery.task -def inform_followers_of_post_update_task(post_id: int, sending_instance_id: int): post = Post.query.get(post_id) page_json = post_to_page(post) page_json['updated'] = ap_datetime(utcnow()) diff --git a/app/community/routes.py b/app/community/routes.py index d5e61031..319a469a 100644 --- a/app/community/routes.py +++ b/app/community/routes.py @@ -24,7 +24,7 @@ from app.community.forms import SearchRemoteCommunity, CreateDiscussionForm, Cre EscalateReportForm, ResolveReportForm, CreateVideoForm, CreatePollForm, RetrieveRemotePost, \ EditCommunityWikiPageForm from app.community.util import search_for_community, actor_to_community, \ - save_post, save_icon_file, save_banner_file, send_to_remote_instance, \ + save_icon_file, save_banner_file, send_to_remote_instance, \ delete_post_from_community, delete_post_reply_from_community, community_in_list, find_local_users, tags_from_string, \ allowed_extensions, end_poll_date from app.constants import SUBSCRIPTION_MEMBER, SUBSCRIPTION_OWNER, POST_TYPE_LINK, POST_TYPE_ARTICLE, POST_TYPE_IMAGE, \ @@ -43,6 +43,7 @@ from app.utils import get_setting, render_template, allowlist_html, markdown_to_ community_moderators, communities_banned_from, show_ban_message, recently_upvoted_posts, recently_downvoted_posts, \ blocked_users, languages_for_form, menu_topics, add_to_modlog, \ blocked_communities, remove_tracking_from_link, piefed_markdown_to_lemmy_markdown, ensure_directory_exists +from app.shared.post import make_post from feedgen.feed import FeedGenerator from datetime import timezone, timedelta from copy import copy @@ -630,110 +631,15 @@ def add_post(actor, type): form.language_id.choices = languages_for_form() - if not can_create_post(current_user, community): - abort(401) - if form.validate_on_submit(): community = Community.query.get_or_404(form.communities.data) - if not can_create_post(current_user, community): + try: + uploaded_file = request.files['image_file'] if type == 'image' else None + post = make_post(form, community, post_type, 1, uploaded_file=uploaded_file) + except Exception as ex: + flash(_('Your post was not accepted because %(reason)s', reason=str(ex)), 'error') abort(401) - language = Language.query.get(form.language_id.data) - - request_json = { - 'id': None, - 'object': { - 'name': form.title.data, - 'type': 'Page', - 'stickied': form.sticky.data, - 'sensitive': form.nsfw.data, - 'nsfl': form.nsfl.data, - 'id': gibberish(), # this will be updated once we have the post.id - 'mediaType': 'text/markdown', - 'content': form.body.data, - 'tag': tags_from_string(form.tags.data), - 'language': {'identifier': language.code, 'name': language.name} - } - } - if type == 'link': - request_json['object']['attachment'] = [{'type': 'Link', 'href': form.link_url.data}] - elif type == 'image': - uploaded_file = request.files['image_file'] - if uploaded_file and uploaded_file.filename != '': - # check if this is an allowed type of file - file_ext = os.path.splitext(uploaded_file.filename)[1] - if file_ext.lower() not in allowed_extensions: - abort(400, description="Invalid image type.") - - new_filename = gibberish(15) - # set up the storage directory - directory = 'app/static/media/posts/' + new_filename[0:2] + '/' + new_filename[2:4] - ensure_directory_exists(directory) - - final_place = os.path.join(directory, new_filename + file_ext) - uploaded_file.seek(0) - uploaded_file.save(final_place) - - if file_ext.lower() == '.heic': - register_heif_opener() - if file_ext.lower() == '.avif': - import pillow_avif - - Image.MAX_IMAGE_PIXELS = 89478485 - - # resize if necessary - if not final_place.endswith('.svg'): - img = Image.open(final_place) - if '.' + img.format.lower() in allowed_extensions: - img = ImageOps.exif_transpose(img) - - # limit full sized version to 2000px - img.thumbnail((2000, 2000)) - img.save(final_place) - - request_json['object']['attachment'] = [{ - 'type': 'Image', - 'url': f'https://{current_app.config["SERVER_NAME"]}/{final_place.replace("app/", "")}', - 'name': form.image_alt_text.data, - 'file_path': final_place - }] - - elif type == 'video': - request_json['object']['attachment'] = [{'type': 'Document', 'url': form.video_url.data}] - elif type == 'poll': - request_json['object']['type'] = 'Question' - choices = [form.choice_1, form.choice_2, form.choice_3, form.choice_4, form.choice_5, - form.choice_6, form.choice_7, form.choice_8, form.choice_9, form.choice_10] - key = 'oneOf' if form.mode.data == 'single' else 'anyOf' - request_json['object'][key] = [] - for choice in choices: - choice_data = choice.data.strip() - if choice_data: - request_json['object'][key].append({'name': choice_data}) - request_json['object']['endTime'] = end_poll_date(form.finish_in.data) - - # todo: add try..except - post = Post.new(current_user, community, request_json) - - if form.notify_author.data: - new_notification = NotificationSubscription(name=post.title, user_id=current_user.id, entity_id=post.id, type=NOTIF_POST) - db.session.add(new_notification) - current_user.language_id = form.language_id.data - g.site.last_active = utcnow() - post.ap_id = f"https://{current_app.config['SERVER_NAME']}/post/{post.id}" - db.session.commit() - - if post.type == POST_TYPE_POLL: - poll = Poll.query.filter_by(post_id=post.id).first() - if not poll.local_only: - federate_post_to_user_followers(post) - if not community.local_only and not poll.local_only: - federate_post(community, post) - else: - federate_post_to_user_followers(post) - if not community.local_only: - federate_post(community, post) - resp = make_response(redirect(f"/post/{post.id}")) # remove cookies used to maintain state when switching post type resp.delete_cookie('post_title') @@ -760,7 +666,6 @@ def add_post(actor, type): form.language_id.data = source_post.language_id form.link_url.data = source_post.url - # empty post to pass since add_post.html extends edit_post.html # and that one checks for a post.image_id for editing image posts post = None @@ -775,197 +680,6 @@ def add_post(actor, type): ) -def federate_post(community, post): - page = { - 'type': 'Page', - 'id': post.ap_id, - 'attributedTo': current_user.public_url(), - 'to': [ - community.public_url(), - 'https://www.w3.org/ns/activitystreams#Public' - ], - 'name': post.title, - 'cc': [], - 'content': post.body_html if post.body_html else '', - 'mediaType': 'text/html', - 'source': {'content': post.body if post.body else '', 'mediaType': 'text/markdown'}, - 'attachment': [], - 'commentsEnabled': post.comments_enabled, - 'sensitive': post.nsfw, - 'nsfl': post.nsfl, - 'stickied': post.sticky, - 'published': ap_datetime(utcnow()), - 'audience': community.public_url(), - 'language': { - 'identifier': post.language_code(), - 'name': post.language_name() - }, - 'tag': post.tags_for_activitypub() - } - create = { - "id": f"https://{current_app.config['SERVER_NAME']}/activities/create/{gibberish(15)}", - "actor": current_user.public_url(), - "to": [ - "https://www.w3.org/ns/activitystreams#Public" - ], - "cc": [ - community.public_url() - ], - "type": "Create", - "audience": community.public_url(), - "object": page, - '@context': default_context() - } - if post.type == POST_TYPE_LINK or post.type == POST_TYPE_VIDEO: - page['attachment'] = [{'href': post.url, 'type': 'Link'}] - elif post.image_id: - image_url = '' - if post.image.source_url: - image_url = post.image.source_url - elif post.image.file_path: - image_url = post.image.file_path.replace('app/static/', - f"https://{current_app.config['SERVER_NAME']}/static/") - elif post.image.thumbnail_path: - image_url = post.image.thumbnail_path.replace('app/static/', - f"https://{current_app.config['SERVER_NAME']}/static/") - # NB image is a dict while attachment is a list of dicts (usually just one dict in the list) - page['image'] = {'type': 'Image', 'url': image_url} - if post.type == POST_TYPE_IMAGE: - page['attachment'] = [{'type': 'Image', - 'url': post.image.source_url, # source_url is always a https link, no need for .replace() as done above - 'name': post.image.alt_text}] - - if post.type == POST_TYPE_POLL: - poll = Poll.query.filter_by(post_id=post.id).first() - page['type'] = 'Question' - page['endTime'] = ap_datetime(poll.end_poll) - page['votersCount'] = 0 - choices = [] - for choice in PollChoice.query.filter_by(post_id=post.id).all(): - choices.append({ - "type": "Note", - "name": choice.choice_text, - "replies": { - "type": "Collection", - "totalItems": 0 - } - }) - page['oneOf' if poll.mode == 'single' else 'anyOf'] = choices - if not community.is_local(): # this is a remote community - send the post to the instance that hosts it - post_request_in_background(community.ap_inbox_url, create, current_user.private_key, - current_user.public_url() + '#main-key', timeout=10) - flash(_('Your post to %(name)s has been made.', name=community.title)) - else: # local community - send (announce) post out to followers - announce = { - "id": f"https://{current_app.config['SERVER_NAME']}/activities/announce/{gibberish(15)}", - "type": 'Announce', - "to": [ - "https://www.w3.org/ns/activitystreams#Public" - ], - "actor": community.public_url(), - "cc": [ - community.ap_followers_url - ], - '@context': default_context(), - 'object': create - } - microblog_announce = copy(announce) - microblog_announce['object'] = post.ap_id - - sent_to = 0 - for instance in community.following_instances(): - if instance.inbox and not current_user.has_blocked_instance(instance.id) and not instance_banned( - instance.domain): - if instance.software in MICROBLOG_APPS: - send_to_remote_instance(instance.id, community.id, microblog_announce) - else: - send_to_remote_instance(instance.id, community.id, announce) - sent_to += 1 - if sent_to: - flash(_('Your post to %(name)s has been made.', name=community.title)) - else: - flash(_('Your post to %(name)s has been made.', name=community.title)) - - -def federate_post_to_user_followers(post): - followers = UserFollower.query.filter_by(local_user_id=post.user_id) - if not followers: - return - - note = { - 'type': 'Note', - 'id': post.ap_id, - 'inReplyTo': None, - 'attributedTo': current_user.public_url(), - 'to': [ - 'https://www.w3.org/ns/activitystreams#Public' - ], - 'cc': [ - current_user.followers_url() - ], - 'content': '', - 'mediaType': 'text/html', - 'attachment': [], - 'commentsEnabled': post.comments_enabled, - 'sensitive': post.nsfw, - 'nsfl': post.nsfl, - 'stickied': post.sticky, - 'published': ap_datetime(utcnow()), - 'language': { - 'identifier': post.language_code(), - 'name': post.language_name() - }, - 'tag': post.tags_for_activitypub() - } - create = { - "id": f"https://{current_app.config['SERVER_NAME']}/activities/create/{gibberish(15)}", - "actor": current_user.public_url(), - "to": [ - "https://www.w3.org/ns/activitystreams#Public" - ], - "cc": [ - current_user.followers_url() - ], - "type": "Create", - "object": note, - '@context': default_context() - } - if post.type == POST_TYPE_ARTICLE: - note['content'] = '

' + post.title + '

' - elif post.type == POST_TYPE_LINK or post.type == POST_TYPE_VIDEO: - note['content'] = '

' + post.title + '

' - elif post.type == POST_TYPE_IMAGE: - note['content'] = '

' + post.title + '

' - if post.image_id and post.image.source_url: - note['attachment'] = [{'type': 'Image', 'url': post.image.source_url, 'name': post.image.alt_text}] - - if post.body_html: - note['content'] = note['content'] + '

' + post.body_html + '

' - - if post.type == POST_TYPE_POLL: - poll = Poll.query.filter_by(post_id=post.id).first() - note['type'] = 'Question' - note['endTime'] = ap_datetime(poll.end_poll) - note['votersCount'] = 0 - choices = [] - for choice in PollChoice.query.filter_by(post_id=post.id).all(): - choices.append({ - "type": "Note", - "name": choice.choice_text, - "replies": { - "type": "Collection", - "totalItems": 0 - } - }) - note['oneOf' if poll.mode == 'single' else 'anyOf'] = choices - - instances = Instance.query.join(User, User.instance_id == Instance.id).join(UserFollower, UserFollower.remote_user_id == User.id) - instances = instances.filter(UserFollower.local_user_id == post.user_id).filter(Instance.gone_forever == False) - for instance in instances: - if instance.inbox and not current_user.has_blocked_instance(instance.id) and not instance_banned(instance.domain): - post_request_in_background(instance.inbox, create, current_user.private_key, current_user.public_url() + '#main-key') - - @bp.route('/community//report', methods=['GET', 'POST']) @login_required def community_report(community_id: int): diff --git a/app/community/util.py b/app/community/util.py index 186f59e9..11f54780 100644 --- a/app/community/util.py +++ b/app/community/util.py @@ -235,233 +235,6 @@ def actor_to_community(actor) -> Community: return community -def save_post(form, post: Post, type: int): - post.indexable = current_user.indexable - post.sticky = form.sticky.data - post.nsfw = form.nsfw.data - post.nsfl = form.nsfl.data - post.notify_author = form.notify_author.data - post.language_id = form.language_id.data - current_user.language_id = form.language_id.data - post.title = form.title.data.strip() - post.body = piefed_markdown_to_lemmy_markdown(form.body.data) - post.body_html = markdown_to_html(post.body) - if not type or type == POST_TYPE_ARTICLE: - post.type = POST_TYPE_ARTICLE - elif type == POST_TYPE_LINK: - url_changed = post.id is None or form.link_url.data != post.url - post.url = remove_tracking_from_link(form.link_url.data.strip()) - post.type = POST_TYPE_LINK - domain = domain_from_url(form.link_url.data) - domain.post_count += 1 - post.domain = domain - - if url_changed: - if post.image_id: - remove_old_file(post.image_id) - post.image_id = None - - if post.url.endswith('.mp4') or post.url.endswith('.webm'): - post.type = POST_TYPE_VIDEO - file = File(source_url=form.link_url.data) # make_image_sizes() will take care of turning this into a still image - post.image = file - db.session.add(file) - else: - unused, file_extension = os.path.splitext(form.link_url.data) - # this url is a link to an image - turn it into a image post - if file_extension.lower() in allowed_extensions: - file = File(source_url=form.link_url.data) - post.image = file - db.session.add(file) - post.type = POST_TYPE_IMAGE - else: - # check opengraph tags on the page and make a thumbnail if an image is available in the og:image meta tag - if not post.type == POST_TYPE_VIDEO: - tn_url = form.link_url.data - if tn_url[:32] == 'https://www.youtube.com/watch?v=': - tn_url = 'https://youtu.be/' + tn_url[32:43] # better chance of thumbnail from youtu.be than youtube.com - opengraph = opengraph_parse(tn_url) - if opengraph and (opengraph.get('og:image', '') != '' or opengraph.get('og:image:url', '') != ''): - filename = opengraph.get('og:image') or opengraph.get('og:image:url') - if not filename.startswith('/'): - file = url_to_thumbnail_file(filename) - if file: - file.alt_text = shorten_string(opengraph.get('og:title'), 295) - post.image = file - db.session.add(file) - - elif type == POST_TYPE_IMAGE: - post.type = POST_TYPE_IMAGE - alt_text = form.image_alt_text.data if form.image_alt_text.data else form.title.data - uploaded_file = request.files['image_file'] - # If we are uploading new file in the place of existing one just remove the old one - if post.image_id is not None and uploaded_file: - post.image.delete_from_disk() - image_id = post.image_id - post.image_id = None - db.session.add(post) - db.session.commit() - File.query.filter_by(id=image_id).delete() - - if uploaded_file and uploaded_file.filename != '': - if post.image_id: - remove_old_file(post.image_id) - post.image_id = None - - # check if this is an allowed type of file - file_ext = os.path.splitext(uploaded_file.filename)[1] - if file_ext.lower() not in allowed_extensions: - abort(400) - new_filename = gibberish(15) - - # set up the storage directory - directory = 'app/static/media/posts/' + new_filename[0:2] + '/' + new_filename[2:4] - ensure_directory_exists(directory) - - # save the file - final_place = os.path.join(directory, new_filename + file_ext) - final_place_medium = os.path.join(directory, new_filename + '_medium.webp') - final_place_thumbnail = os.path.join(directory, new_filename + '_thumbnail.webp') - uploaded_file.seek(0) - uploaded_file.save(final_place) - - if file_ext.lower() == '.heic': - register_heif_opener() - - Image.MAX_IMAGE_PIXELS = 89478485 - - # resize if necessary - img = Image.open(final_place) - if '.' + img.format.lower() in allowed_extensions: - img = ImageOps.exif_transpose(img) - - # limit full sized version to 2000px - img_width = img.width - img_height = img.height - img.thumbnail((2000, 2000)) - img.save(final_place) - - # medium sized version - img.thumbnail((512, 512)) - img.save(final_place_medium, format="WebP", quality=93) - - # save a third, smaller, version as a thumbnail - img.thumbnail((170, 170)) - img.save(final_place_thumbnail, format="WebP", quality=93) - thumbnail_width = img.width - thumbnail_height = img.height - - file = File(file_path=final_place_medium, file_name=new_filename + file_ext, alt_text=alt_text, - width=img_width, height=img_height, thumbnail_width=thumbnail_width, - thumbnail_height=thumbnail_height, thumbnail_path=final_place_thumbnail, - source_url=final_place.replace('app/static/', f"https://{current_app.config['SERVER_NAME']}/static/")) - db.session.add(file) - db.session.commit() - post.image_id = file.id - - elif type == POST_TYPE_VIDEO: - form.video_url.data = form.video_url.data.strip() - url_changed = post.id is None or form.video_url.data != post.url - post.url = remove_tracking_from_link(form.video_url.data.strip()) - post.type = POST_TYPE_VIDEO - domain = domain_from_url(form.video_url.data) - domain.post_count += 1 - post.domain = domain - - if url_changed: - if post.image_id: - remove_old_file(post.image_id) - post.image_id = None - if form.video_url.data.endswith('.mp4') or form.video_url.data.endswith('.webm'): - file = File(source_url=form.video_url.data) # make_image_sizes() will take care of turning this into a still image - post.image = file - db.session.add(file) - else: - # check opengraph tags on the page and make a thumbnail if an image is available in the og:image meta tag - tn_url = form.video_url.data - if tn_url[:32] == 'https://www.youtube.com/watch?v=': - tn_url = 'https://youtu.be/' + tn_url[32:43] # better chance of thumbnail from youtu.be than youtube.com - opengraph = opengraph_parse(tn_url) - if opengraph and (opengraph.get('og:image', '') != '' or opengraph.get('og:image:url', '') != ''): - filename = opengraph.get('og:image') or opengraph.get('og:image:url') - if not filename.startswith('/'): - file = url_to_thumbnail_file(filename) - if file: - file.alt_text = shorten_string(opengraph.get('og:title'), 295) - post.image = file - db.session.add(file) - - elif type == POST_TYPE_POLL: - post.body = form.title.data + '\n' + form.body.data if post.title not in form.body.data else form.body.data - post.body_html = markdown_to_html(post.body) - post.type = POST_TYPE_POLL - else: - raise Exception('invalid post type') - - if post.id is None: - if current_user.reputation > 100: - post.up_votes = 1 - post.score = 1 - if current_user.reputation < -100: - post.score = -1 - post.ranking = post.post_ranking(post.score, utcnow()) - - # Filter by phrase - blocked_phrases_list = blocked_phrases() - for blocked_phrase in blocked_phrases_list: - if blocked_phrase in post.title: - abort(401) - return - if post.body: - for blocked_phrase in blocked_phrases_list: - if blocked_phrase in post.body: - abort(401) - return - - db.session.add(post) - else: - db.session.execute(text('DELETE FROM "post_tag" WHERE post_id = :post_id'), {'post_id': post.id}) - post.tags = tags_from_string_old(form.tags.data) - db.session.commit() - - # Save poll choices. NB this will delete all votes whenever a poll is edited. Partially because it's easier to code but also to stop malicious alterations to polls after people have already voted - if type == POST_TYPE_POLL: - db.session.execute(text('DELETE FROM "poll_choice_vote" WHERE post_id = :post_id'), {'post_id': post.id}) - db.session.execute(text('DELETE FROM "poll_choice" WHERE post_id = :post_id'), {'post_id': post.id}) - for i in range(1, 10): - choice_data = getattr(form, f"choice_{i}").data.strip() - if choice_data != '': - db.session.add(PollChoice(post_id=post.id, choice_text=choice_data, sort_order=i)) - - poll = Poll.query.filter_by(post_id=post.id).first() - if poll is None: - poll = Poll(post_id=post.id) - db.session.add(poll) - poll.mode = form.mode.data - if form.finish_in: - poll.end_poll = end_poll_date(form.finish_in.data) - poll.local_only = form.local_only.data - poll.latest_vote = None - db.session.commit() - - # Notify author about replies - # Remove any subscription that currently exists - existing_notification = NotificationSubscription.query.filter(NotificationSubscription.entity_id == post.id, - NotificationSubscription.user_id == current_user.id, - NotificationSubscription.type == NOTIF_POST).first() - if existing_notification: - db.session.delete(existing_notification) - - # Add subscription if necessary - if form.notify_author.data: - new_notification = NotificationSubscription(name=post.title, user_id=current_user.id, entity_id=post.id, - type=NOTIF_POST) - db.session.add(new_notification) - - g.site.last_active = utcnow() - db.session.commit() - - def end_poll_date(end_choice): delta_mapping = { '30m': timedelta(minutes=30), diff --git a/app/post/routes.py b/app/post/routes.py index dee6a2b9..089b4a25 100644 --- a/app/post/routes.py +++ b/app/post/routes.py @@ -10,8 +10,8 @@ from wtforms import SelectField, RadioField from app import db, constants, cache, celery from app.activitypub.signature import HttpSignature, post_request, default_context, post_request_in_background -from app.activitypub.util import notify_about_post_reply, inform_followers_of_post_update, update_post_from_activity -from app.community.util import save_post, send_to_remote_instance +from app.activitypub.util import notify_about_post_reply, update_post_from_activity +from app.community.util import send_to_remote_instance from app.inoculation import inoculation from app.post.forms import NewReplyForm, ReportPostForm, MeaCulpaForm, CrossPostForm from app.community.forms import CreateLinkForm, CreateImageForm, CreateDiscussionForm, CreateVideoForm, CreatePollForm, EditImageForm @@ -35,6 +35,7 @@ from app.utils import get_setting, render_template, allowlist_html, markdown_to_ languages_for_form, menu_topics, add_to_modlog, blocked_communities, piefed_markdown_to_lemmy_markdown, \ permission_required, blocked_users, get_request, is_local_image_url, is_video_url, can_upvote, can_downvote from app.shared.reply import make_reply, edit_reply +from app.shared.post import edit_post def show_post(post_id: int): @@ -359,7 +360,8 @@ def poll_vote(post_id): poll_votes = PollChoice.query.join(PollChoiceVote, PollChoiceVote.choice_id == PollChoice.id).filter(PollChoiceVote.post_id == post.id, PollChoiceVote.user_id == current_user.id).all() for pv in poll_votes: if post.author.is_local(): - inform_followers_of_post_update(post.id, 1) + from app.shared.tasks import task_selector + task_selector('edit_post', user_id=current_user.id, post_id=post.id) else: pollvote_json = { '@context': default_context(), @@ -587,7 +589,7 @@ def post_edit(post_id: int): del form.finish_in else: abort(404) - + del form.communities mods = post.community.moderators() @@ -607,26 +609,16 @@ def post_edit(post_id: int): form.nsfl.data = True form.nsfw.render_kw = {'disabled': True} - old_url = post.url - form.language_id.choices = languages_for_form() if form.validate_on_submit(): - save_post(form, post, post_type) - post.community.last_active = utcnow() - post.edited_at = utcnow() - - if post.url != old_url: - post.calculate_cross_posts(url_changed=True) - - db.session.commit() - - flash(_('Your changes have been saved.'), 'success') - - # federate edit - if not post.community.local_only: - federate_post_update(post) - federate_post_edit_to_user_followers(post) + try: + uploaded_file = request.files['image_file'] if post_type == POST_TYPE_IMAGE else None + edit_post(form, post, post_type, 1, uploaded_file=uploaded_file) + flash(_('Your changes have been saved.'), 'success') + except Exception as ex: + flash(_('Your edit was not accepted because %(reason)s', reason=str(ex)), 'error') + abort(401) return redirect(url_for('activitypub.post_ap', post_id=post.id)) else: @@ -651,7 +643,7 @@ def post_edit(post_id: int): ) with open(path, "rb")as file: form.image_file.data = file.read() - + elif post_type == POST_TYPE_VIDEO: form.video_url.data = post.url elif post_type == POST_TYPE_POLL: @@ -678,186 +670,6 @@ def post_edit(post_id: int): abort(401) -def federate_post_update(post): - page_json = { - 'type': 'Page', - 'id': post.ap_id, - 'attributedTo': current_user.public_url(), - 'to': [ - post.community.public_url(), - 'https://www.w3.org/ns/activitystreams#Public' - ], - 'name': post.title, - 'cc': [], - 'content': post.body_html if post.body_html else '', - 'mediaType': 'text/html', - 'source': {'content': post.body if post.body else '', 'mediaType': 'text/markdown'}, - 'attachment': [], - 'commentsEnabled': post.comments_enabled, - 'sensitive': post.nsfw, - 'nsfl': post.nsfl, - 'stickied': post.sticky, - 'published': ap_datetime(post.posted_at), - 'updated': ap_datetime(post.edited_at), - 'audience': post.community.public_url(), - 'language': { - 'identifier': post.language_code(), - 'name': post.language_name() - }, - 'tag': post.tags_for_activitypub() - } - update_json = { - 'id': f"https://{current_app.config['SERVER_NAME']}/activities/update/{gibberish(15)}", - 'type': 'Update', - 'actor': current_user.public_url(), - 'audience': post.community.public_url(), - 'to': [post.community.public_url(), 'https://www.w3.org/ns/activitystreams#Public'], - 'published': ap_datetime(utcnow()), - 'cc': [ - current_user.followers_url() - ], - 'object': page_json, - } - if post.type == POST_TYPE_LINK or post.type == POST_TYPE_VIDEO: - page_json['attachment'] = [{'href': post.url, 'type': 'Link'}] - elif post.image_id: - if post.image.file_path: - image_url = post.image.file_path.replace('app/static/', - f"https://{current_app.config['SERVER_NAME']}/static/") - elif post.image.thumbnail_path: - image_url = post.image.thumbnail_path.replace('app/static/', - f"https://{current_app.config['SERVER_NAME']}/static/") - else: - image_url = post.image.source_url - # NB image is a dict while attachment is a list of dicts (usually just one dict in the list) - page_json['image'] = {'type': 'Image', 'url': image_url} - if post.type == POST_TYPE_IMAGE: - page_json['attachment'] = [{'type': 'Image', - 'url': post.image.source_url, # source_url is always a https link, no need for .replace() as done above - 'name': post.image.alt_text}] - if post.type == POST_TYPE_POLL: - poll = Poll.query.filter_by(post_id=post.id).first() - page_json['type'] = 'Question' - page_json['endTime'] = ap_datetime(poll.end_poll) - page_json['votersCount'] = 0 - choices = [] - for choice in PollChoice.query.filter_by(post_id=post.id).all(): - choices.append({ - "type": "Note", - "name": choice.choice_text, - "replies": { - "type": "Collection", - "totalItems": 0 - } - }) - page_json['oneOf' if poll.mode == 'single' else 'anyOf'] = choices - - if not post.community.is_local(): # this is a remote community, send it to the instance that hosts it - success = post_request(post.community.ap_inbox_url, update_json, current_user.private_key, - current_user.public_url() + '#main-key') - if success is False or isinstance(success, str): - flash('Failed to send edit to remote server', 'error') - else: # local community - send it to followers on remote instances - announce = { - "id": f"https://{current_app.config['SERVER_NAME']}/activities/announce/{gibberish(15)}", - "type": 'Announce', - "to": [ - "https://www.w3.org/ns/activitystreams#Public" - ], - "actor": post.community.ap_profile_id, - "cc": [ - post.community.ap_followers_url - ], - '@context': default_context(), - 'object': update_json - } - - for instance in post.community.following_instances(): - if instance.inbox and not current_user.has_blocked_instance(instance.id) and not instance_banned( - instance.domain): - send_to_remote_instance(instance.id, post.community.id, announce) - - -def federate_post_edit_to_user_followers(post): - followers = UserFollower.query.filter_by(local_user_id=post.user_id) - if not followers: - return - - note = { - 'type': 'Note', - 'id': post.ap_id, - 'inReplyTo': None, - 'attributedTo': current_user.public_url(), - 'to': [ - 'https://www.w3.org/ns/activitystreams#Public' - ], - 'cc': [ - current_user.followers_url() - ], - 'content': '', - 'mediaType': 'text/html', - 'source': {'content': post.body if post.body else '', 'mediaType': 'text/markdown'}, - 'attachment': [], - 'commentsEnabled': post.comments_enabled, - 'sensitive': post.nsfw, - 'nsfl': post.nsfl, - 'stickied': post.sticky, - 'published': ap_datetime(utcnow()), - 'updated': ap_datetime(post.edited_at), - 'language': { - 'identifier': post.language_code(), - 'name': post.language_name() - }, - 'tag': post.tags_for_activitypub() - } - update = { - "id": f"https://{current_app.config['SERVER_NAME']}/activities/create/{gibberish(15)}", - "actor": current_user.public_url(), - "to": [ - "https://www.w3.org/ns/activitystreams#Public" - ], - "cc": [ - current_user.followers_url() - ], - "type": "Update", - "object": note, - '@context': default_context() - } - if post.type == POST_TYPE_ARTICLE: - note['content'] = '

' + post.title + '

' - elif post.type == POST_TYPE_LINK or post.type == POST_TYPE_VIDEO: - note['content'] = '

' + post.title + '

' - elif post.type == POST_TYPE_IMAGE: - note['content'] = '

' + post.title + '

' - if post.image_id and post.image.source_url: - note['attachment'] = [{'type': 'Image', 'url': post.image.source_url, 'name': post.image.alt_text}] - elif post.type == POST_TYPE_POLL: - poll = Poll.query.filter_by(post_id=post.id).first() - note['type'] = 'Question' - note['endTime'] = ap_datetime(poll.end_poll) - note['votersCount'] = 0 - choices = [] - for choice in PollChoice.query.filter_by(post_id=post.id).all(): - choices.append({ - "type": "Note", - "name": choice.choice_text, - "replies": { - "type": "Collection", - "totalItems": 0 - } - }) - note['oneOf' if poll.mode == 'single' else 'anyOf'] = choices - - if post.body_html: - note['content'] = note['content'] + '

' + post.body_html + '

' - - instances = Instance.query.join(User, User.instance_id == Instance.id).join(UserFollower, UserFollower.remote_user_id == User.id) - instances = instances.filter(UserFollower.local_user_id == post.user_id) - for instance in instances: - if instance.inbox and not instance_banned(instance.domain): - post_request_in_background(instance.inbox, update, current_user.private_key, current_user.public_url() + '#main-key') - - @bp.route('/post//delete', methods=['GET', 'POST']) @login_required def post_delete(post_id: int): From 6a4e89149f1ec0d85818df9a68427cb98e6186a9 Mon Sep 17 00:00:00 2001 From: freamon Date: Fri, 17 Jan 2025 17:41:24 +0000 Subject: [PATCH 08/19] mentions in post bodies - recognise @user@instance syntax in UI --- app/templates/post/_post_full.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/templates/post/_post_full.html b/app/templates/post/_post_full.html index bc8cf622..4fefa7d1 100644 --- a/app/templates/post/_post_full.html +++ b/app/templates/post/_post_full.html @@ -100,7 +100,7 @@ {% endif -%} {% endif -%}
- {{ post.body_html|community_links|safe if post.body_html else '' }} + {{ post.body_html | community_links | person_links | safe if post.body_html else '' }} {% if archive_link -%}

{{ _('Archive.ph link') }}

{% endif -%} From 022d1d67dcf965bc09509e88aa189e3cd021def4 Mon Sep 17 00:00:00 2001 From: freamon Date: Fri, 17 Jan 2025 18:35:13 +0000 Subject: [PATCH 09/19] mentions in post bodies - inbound federation --- app/activitypub/util.py | 40 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/app/activitypub/util.py b/app/activitypub/util.py index 898a509c..040a468a 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -1571,8 +1571,8 @@ def create_post_reply(store_ap_json, community: Community, in_reply_to, request_ # Check for Mentions of local users reply_parent = parent_comment if parent_comment else post local_users_to_notify = [] - if 'tag' in request_json and isinstance(request_json['tag'], list): - for json_tag in request_json['tag']: + if 'tag' in request_json['object'] and isinstance(request_json['object']['tag'], list): + for json_tag in request_json['object']['tag']: if 'type' in json_tag and json_tag['type'] == 'Mention': profile_id = json_tag['href'] if 'href' in json_tag else None if profile_id and isinstance(profile_id, str) and profile_id.startswith('https://' + current_app.config['SERVER_NAME']): @@ -1612,6 +1612,23 @@ def create_post(store_ap_json, community: Community, request_json: dict, user: U return None try: post = Post.new(user, community, request_json, announce_id) + # can't do this in app.models, 'cos can't import blocked_users + if 'tag' in request_json['object'] and isinstance(request_json['object']['tag'], list): + for json_tag in request_json['object']['tag']: + if 'type' in json_tag and json_tag['type'] == 'Mention': + profile_id = json_tag['href'] if 'href' in json_tag else None + if profile_id and isinstance(profile_id, str) and profile_id.startswith('https://' + current_app.config['SERVER_NAME']): + profile_id = profile_id.lower() + recipient = User.query.filter_by(ap_profile_id=profile_id, ap_id=None).first() + if recipient: + blocked_senders = blocked_users(recipient.id) + if post.user_id not in blocked_senders: + notification = Notification(user_id=recipient.id, title=_(f"You have been mentioned in post {post.id}"), + url=f"https://{current_app.config['SERVER_NAME']}/post/{post.id}", + author_id=post.user_id) + recipient.unread_notifications += 1 + db.session.add(notification) + db.session.commit() return post except Exception as ex: log_incoming_ap(id, APLOG_CREATE, APLOG_FAILURE, saved_json, str(ex)) @@ -1690,8 +1707,8 @@ def update_post_reply_from_activity(reply: PostReply, request_json: dict): reply.edited_at = utcnow() # Check for Mentions of local users (that weren't in the original) - if 'tag' in request_json and isinstance(request_json['tag'], list): - for json_tag in request_json['tag']: + if 'tag' in request_json['object'] and isinstance(request_json['object']['tag'], list): + for json_tag in request_json['object']['tag']: if 'type' in json_tag and json_tag['type'] == 'Mention': profile_id = json_tag['href'] if 'href' in json_tag else None if profile_id and isinstance(profile_id, str) and profile_id.startswith('https://' + current_app.config['SERVER_NAME']): @@ -1778,6 +1795,21 @@ def update_post_from_activity(post: Post, request_json: dict): hashtag = find_hashtag_or_create(json_tag['name']) if hashtag: post.tags.append(hashtag) + if 'type' in json_tag and json_tag['type'] == 'Mention': + profile_id = json_tag['href'] if 'href' in json_tag else None + if profile_id and isinstance(profile_id, str) and profile_id.startswith('https://' + current_app.config['SERVER_NAME']): + profile_id = profile_id.lower() + recipient = User.query.filter_by(ap_profile_id=profile_id, ap_id=None).first() + if recipient: + blocked_senders = blocked_users(recipient.id) + if post.user_id not in blocked_senders: + existing_notification = Notification.query.filter(Notification.user_id == recipient.id, Notification.url == f"https://{current_app.config['SERVER_NAME']}/post/{post.id}").first() + if not existing_notification: + notification = Notification(user_id=recipient.id, title=_(f"You have been mentioned in post {post.id}"), + url=f"https://{current_app.config['SERVER_NAME']}/post/{post.id}", + author_id=post.user_id) + recipient.unread_notifications += 1 + db.session.add(notification) post.comments_enabled = request_json['object']['commentsEnabled'] if 'commentsEnabled' in request_json['object'] else True post.edited_at = utcnow() From 33dcbd45b01fb8301fc938eed641892ff5432041 Mon Sep 17 00:00:00 2001 From: freamon Date: Fri, 17 Jan 2025 19:59:45 +0000 Subject: [PATCH 10/19] outgoing federation: remove tags from Create / Update level --- app/shared/tasks/notes.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/app/shared/tasks/notes.py b/app/shared/tasks/notes.py index 1d38aa68..93a029fe 100644 --- a/app/shared/tasks/notes.py +++ b/app/shared/tasks/notes.py @@ -42,7 +42,6 @@ import re 'cc': [] '@context': (outer object only) 'audience': (not in Announce) - 'tag': [] (not in Announce) } """ @@ -162,8 +161,7 @@ def send_reply(user_id, reply_id, parent_id, edit=False): 'to': to, 'cc': cc, '@context': default_context(), - 'audience': community.public_url(), - 'tag': tag + 'audience': community.public_url() } domains_sent_to = [current_app.config['SERVER_NAME']] From 6df39c02f35a539cd1b22065ab5dc9aec9bb716f Mon Sep 17 00:00:00 2001 From: freamon Date: Fri, 17 Jan 2025 22:23:08 +0000 Subject: [PATCH 11/19] outgoing federation: improve inter-op with microblog apps for posts --- app/shared/tasks/pages.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/app/shared/tasks/pages.py b/app/shared/tasks/pages.py index e88701db..d1116c34 100644 --- a/app/shared/tasks/pages.py +++ b/app/shared/tasks/pages.py @@ -23,6 +23,7 @@ import re 'content': 'mediaType': 'source': {} + 'inReplyTo': (only added for Mentions and user followers, as Note with inReplyTo: None) 'published': 'updated': (inner oject of Update only) 'language': {} @@ -48,7 +49,6 @@ import re 'cc': [] '@context': (outer object only) 'audience': (not in Announce) - 'tag': [] (not in Announce) } """ @@ -118,7 +118,7 @@ def send_post(user_id, post_id, edit=False): # local_only communities can also be used to send activity to User Followers # return now though, if there aren't any - followers = UserFollower.query.filter_by(local_user_id=post.user_id).first() + followers = UserFollower.query.filter_by(local_user_id=post.user_id).all() if not followers and community.local_only: return @@ -197,8 +197,7 @@ def send_post(user_id, post_id, edit=False): 'to': to, 'cc': cc, '@context': default_context(), - 'audience': community.public_url(), - 'tag': tag + 'audience': community.public_url() } domains_sent_to = [current_app.config['SERVER_NAME']] @@ -247,15 +246,9 @@ def send_post(user_id, post_id, edit=False): post_request(community.ap_inbox_url, create, user.private_key, user.public_url() + '#main-key') domains_sent_to.append(community.instance.domain) - # send copy of the Create to anyone else Mentioned in post, but not on an instance that's already sent to. - if '@context' not in create: - create['@context'] = default_context() - for recipient in recipients: - if recipient.instance.domain not in domains_sent_to: - post_request(recipient.instance.inbox, create, user.private_key, user.public_url() + '#main-key') - domains_sent_to.append(recipient.instance.domain) - - # send amended copy of the Create to anyone who is following the User, but hasn't already received something + # amend copy of the Create, for anyone Mentioned in post body or who is following the user, to a format more likely to be understood + if '@context' not in create: + create['@context'] = default_context() if 'name' in page: del page['name'] note = page @@ -263,13 +256,26 @@ def send_post(user_id, post_id, edit=False): note['type'] = 'Note' if post.type == POST_TYPE_LINK or post.type == POST_TYPE_VIDEO: note['content'] = '

' + post.title + '

' - else: + elif post.type != POST_TYPE_POLL: note['content'] = '

' + post.title + '

' if post.body_html: note['content'] = note['content'] + post.body_html note['inReplyTo'] = None create['object'] = note + if not community.local_only: + for recipient in recipients: + if recipient.instance.domain not in domains_sent_to: + post_request(recipient.instance.inbox, create, user.private_key, user.public_url() + '#main-key') + domains_sent_to.append(recipient.instance.domain) + if not followers: + return + + # send the amended copy of the Create to anyone who is following the User, but hasn't already received something + for follower in followers: + user_details = User.query.get(follower.remote_user_id) + if user_details: + create['cc'].append(user_details.public_url()) instances = Instance.query.join(User, User.instance_id == Instance.id).join(UserFollower, UserFollower.remote_user_id == User.id) instances = instances.filter(UserFollower.local_user_id == post.user_id).filter(Instance.gone_forever == False) for instance in instances: From ec232d9a178a9a4cc48a82ed557b2c00a4add483 Mon Sep 17 00:00:00 2001 From: freamon Date: Fri, 17 Jan 2025 22:47:40 +0000 Subject: [PATCH 12/19] Catch UnidentifiedImageError if non-image uploaded --- app/community/forms.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/community/forms.py b/app/community/forms.py index cd034504..bc5857dd 100644 --- a/app/community/forms.py +++ b/app/community/forms.py @@ -12,7 +12,7 @@ from app.constants import DOWNVOTE_ACCEPT_ALL, DOWNVOTE_ACCEPT_MEMBERS, DOWNVOTE DOWNVOTE_ACCEPT_TRUSTED from app.models import Community, utcnow from app.utils import domain_from_url, MultiCheckboxField -from PIL import Image, ImageOps +from PIL import Image, ImageOps, UnidentifiedImageError from io import BytesIO import pytesseract @@ -173,6 +173,8 @@ class CreateImageForm(CreatePostForm): image_text = pytesseract.image_to_string(Image.open(BytesIO(uploaded_file.read())).convert('L')) except FileNotFoundError as e: image_text = '' + except UnidentifiedImageError as e: + image_text = '' if 'Anonymous' in image_text and ( 'No.' in image_text or ' N0' in image_text): # chan posts usually contain the text 'Anonymous' and ' No.12345' self.image_file.errors.append( From b4aa566dba55796d55c50784b32915cfb6e6e1a1 Mon Sep 17 00:00:00 2001 From: freamon Date: Fri, 17 Jan 2025 23:22:32 +0000 Subject: [PATCH 13/19] Also increment subscriptions_count for remote users joining local communities #424 --- app/activitypub/routes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index 08c08bea..24afb5ce 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -672,6 +672,7 @@ def process_inbox_request(request_json, store_ap_json): if community_membership(user, community) != SUBSCRIPTION_MEMBER: member = CommunityMember(user_id=user.id, community_id=community.id) db.session.add(member) + community.subscriptions_count += 1 db.session.commit() cache.delete_memoized(community_membership, user, community) # send accept message to acknowledge the follow From 1df4cde0d825e33fb804f59fe9fe33c19060d7f8 Mon Sep 17 00:00:00 2001 From: freamon Date: Sat, 18 Jan 2025 00:35:20 +0000 Subject: [PATCH 14/19] API: fixes for post create / edit with the app --- app/api/alpha/utils/post.py | 10 +++++++--- app/shared/post.py | 7 ++++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/app/api/alpha/utils/post.py b/app/api/alpha/utils/post.py index 12510c8d..c5ee29a0 100644 --- a/app/api/alpha/utils/post.py +++ b/app/api/alpha/utils/post.py @@ -167,7 +167,9 @@ def post_post(auth, data): language_id = 2 # change when Polls are supported - type = None + type = POST_TYPE_ARTICLE + if url: + type = POST_TYPE_LINK input = {'title': title, 'body': body, 'url': url, 'nsfw': nsfw, 'language_id': language_id, 'notify_author': True} community = Community.query.filter_by(id=community_id).one() @@ -193,11 +195,13 @@ def put_post(auth, data): language_id = 2 # change when Polls are supported - type = None + type = POST_TYPE_ARTICLE + if url: + type = POST_TYPE_LINK input = {'title': title, 'body': body, 'url': url, 'nsfw': nsfw, 'language_id': language_id, 'notify_author': True} post = Post.query.filter_by(id=post_id).one() - user_id, post = edit_post(input, post, type, SRC_API, auth) + user_id, post = edit_post(input, post, type, SRC_API, auth=auth) post_json = post_view(post=post, variant=4, user_id=user_id) return post_json diff --git a/app/shared/post.py b/app/shared/post.py index 7804fb12..5116b9d8 100644 --- a/app/shared/post.py +++ b/app/shared/post.py @@ -275,9 +275,10 @@ def edit_post(input, post, type, src, user=None, auth=None, uploaded_file=None, if remove_file: remove_file.delete_from_disk() post.image_id = None - domain = domain_from_url(post.url) - if domain: - domain.post_count -= 1 + if post.url: + domain = domain_from_url(post.url) + if domain: + domain.post_count -= 1 # remove any old tags db.session.execute(text('DELETE FROM "post_tag" WHERE post_id = :post_id'), {'post_id': post.id}) From 3e3aca1910e8f86687a0dde416163bde76deba5e Mon Sep 17 00:00:00 2001 From: freamon Date: Sat, 18 Jan 2025 00:59:26 +0000 Subject: [PATCH 15/19] move check for incoming Mentions into post.new() --- app/activitypub/util.py | 17 ----------------- app/models.py | 20 +++++++++++++++++++- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/app/activitypub/util.py b/app/activitypub/util.py index 040a468a..06ff7843 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -1612,23 +1612,6 @@ def create_post(store_ap_json, community: Community, request_json: dict, user: U return None try: post = Post.new(user, community, request_json, announce_id) - # can't do this in app.models, 'cos can't import blocked_users - if 'tag' in request_json['object'] and isinstance(request_json['object']['tag'], list): - for json_tag in request_json['object']['tag']: - if 'type' in json_tag and json_tag['type'] == 'Mention': - profile_id = json_tag['href'] if 'href' in json_tag else None - if profile_id and isinstance(profile_id, str) and profile_id.startswith('https://' + current_app.config['SERVER_NAME']): - profile_id = profile_id.lower() - recipient = User.query.filter_by(ap_profile_id=profile_id, ap_id=None).first() - if recipient: - blocked_senders = blocked_users(recipient.id) - if post.user_id not in blocked_senders: - notification = Notification(user_id=recipient.id, title=_(f"You have been mentioned in post {post.id}"), - url=f"https://{current_app.config['SERVER_NAME']}/post/{post.id}", - author_id=post.user_id) - recipient.unread_notifications += 1 - db.session.add(notification) - db.session.commit() return post except Exception as ex: log_incoming_ap(id, APLOG_CREATE, APLOG_FAILURE, saved_json, str(ex)) diff --git a/app/models.py b/app/models.py index 5c14121a..05017f65 100644 --- a/app/models.py +++ b/app/models.py @@ -1182,7 +1182,7 @@ class Post(db.Model): find_licence_or_create, make_image_sizes, notify_about_post from app.utils import allowlist_html, markdown_to_html, html_to_text, microblog_content_to_title, blocked_phrases, \ is_image_url, is_video_url, domain_from_url, opengraph_parse, shorten_string, remove_tracking_from_link, \ - is_video_hosting_site, communities_banned_from, recently_upvoted_posts + is_video_hosting_site, communities_banned_from, recently_upvoted_posts, blocked_users microblog = False if 'name' not in request_json['object']: # Microblog posts @@ -1376,6 +1376,24 @@ class Post(db.Model): db.session.rollback() return Post.query.filter_by(ap_id=request_json['object']['id'].lower()).one() + # Mentions also need a post_id + if 'tag' in request_json['object'] and isinstance(request_json['object']['tag'], list): + for json_tag in request_json['object']['tag']: + if 'type' in json_tag and json_tag['type'] == 'Mention': + profile_id = json_tag['href'] if 'href' in json_tag else None + if profile_id and isinstance(profile_id, str) and profile_id.startswith('https://' + current_app.config['SERVER_NAME']): + profile_id = profile_id.lower() + recipient = User.query.filter_by(ap_profile_id=profile_id, ap_id=None).first() + if recipient: + blocked_senders = blocked_users(recipient.id) + if post.user_id not in blocked_senders: + notification = Notification(user_id=recipient.id, title=_(f"You have been mentioned in post {post.id}"), + url=f"https://{current_app.config['SERVER_NAME']}/post/{post.id}", + author_id=post.user_id) + recipient.unread_notifications += 1 + db.session.add(notification) + db.session.commit() + # Polls need to be processed quite late because they need a post_id to refer to if request_json['object']['type'] == 'Question': post.type = constants.POST_TYPE_POLL From ac6f66e892d9c78f4123181bd6f4a85da5d6326e Mon Sep 17 00:00:00 2001 From: freamon Date: Sat, 18 Jan 2025 14:52:09 +0000 Subject: [PATCH 16/19] API: post delete and restore --- app/api/alpha/routes.py | 15 ++++++++-- app/api/alpha/utils/__init__.py | 2 +- app/api/alpha/utils/post.py | 19 +++++++++++- app/shared/post.py | 52 +++++++++++++++++++++++++++++++++ app/shared/reply.py | 12 +++----- app/shared/tasks/__init__.py | 6 ++-- app/shared/tasks/deletes.py | 46 ++++++++++++++++++++++++++--- 7 files changed, 134 insertions(+), 18 deletions(-) diff --git a/app/api/alpha/routes.py b/app/api/alpha/routes.py index 5de30055..20c5be37 100644 --- a/app/api/alpha/routes.py +++ b/app/api/alpha/routes.py @@ -1,7 +1,7 @@ from app.api.alpha import bp from app.api.alpha.utils import get_site, post_site_block, \ get_search, \ - get_post_list, get_post, post_post_like, put_post_save, put_post_subscribe, post_post, put_post, \ + get_post_list, get_post, post_post_like, put_post_save, put_post_subscribe, post_post, put_post, post_post_delete, \ get_reply_list, post_reply_like, put_reply_save, put_reply_subscribe, post_reply, put_reply, post_reply_delete, post_reply_report, \ get_community_list, get_community, post_community_follow, post_community_block, \ get_user, post_user_block @@ -183,6 +183,18 @@ def put_alpha_post(): return jsonify({"error": str(ex)}), 400 +@bp.route('/api/alpha/post/delete', methods=['POST']) +def post_alpha_post_delete(): + if not enable_api(): + return jsonify({'error': 'alpha api is not enabled'}) + try: + auth = request.headers.get('Authorization') + data = request.get_json(force=True) or {} + return jsonify(post_post_delete(auth, data)) + except Exception as ex: + return jsonify({"error": str(ex)}), 400 + + # Reply @bp.route('/api/alpha/comment/list', methods=['GET']) def get_alpha_comment_list(): @@ -345,7 +357,6 @@ def alpha_community(): return jsonify({"error": "not_yet_implemented"}), 400 # Post - not yet implemented -@bp.route('/api/alpha/post/delete', methods=['POST']) @bp.route('/api/alpha/post/remove', methods=['POST']) @bp.route('/api/alpha/post/lock', methods=['POST']) @bp.route('/api/alpha/post/feature', methods=['POST']) diff --git a/app/api/alpha/utils/__init__.py b/app/api/alpha/utils/__init__.py index a770a08c..79428210 100644 --- a/app/api/alpha/utils/__init__.py +++ b/app/api/alpha/utils/__init__.py @@ -1,6 +1,6 @@ from app.api.alpha.utils.site import get_site, post_site_block from app.api.alpha.utils.misc import get_search -from app.api.alpha.utils.post import get_post_list, get_post, post_post_like, put_post_save, put_post_subscribe, post_post, put_post +from app.api.alpha.utils.post import get_post_list, get_post, post_post_like, put_post_save, put_post_subscribe, post_post, put_post, post_post_delete from app.api.alpha.utils.reply import get_reply_list, post_reply_like, put_reply_save, put_reply_subscribe, post_reply, put_reply, post_reply_delete, post_reply_report from app.api.alpha.utils.community import get_community, get_community_list, post_community_follow, post_community_block from app.api.alpha.utils.user import get_user, post_user_block diff --git a/app/api/alpha/utils/post.py b/app/api/alpha/utils/post.py index c5ee29a0..3da7dc21 100644 --- a/app/api/alpha/utils/post.py +++ b/app/api/alpha/utils/post.py @@ -3,7 +3,7 @@ from app.api.alpha.views import post_view from app.api.alpha.utils.validators import required, integer_expected, boolean_expected, string_expected from app.constants import POST_TYPE_ARTICLE, POST_TYPE_LINK, POST_TYPE_IMAGE, POST_TYPE_VIDEO from app.models import Post, Community, CommunityMember, utcnow -from app.shared.post import vote_for_post, bookmark_the_post, remove_the_bookmark_from_post, toggle_post_notification, make_post, edit_post +from app.shared.post import vote_for_post, bookmark_the_post, remove_the_bookmark_from_post, toggle_post_notification, make_post, edit_post, delete_post, restore_post from app.utils import authorise_api_user, blocked_users, blocked_communities, blocked_instances, community_ids_from_instances, is_image_url, is_video_url from datetime import timedelta @@ -205,3 +205,20 @@ def put_post(auth, data): post_json = post_view(post=post, variant=4, user_id=user_id) return post_json + + +def post_post_delete(auth, data): + required(['post_id', 'deleted'], data) + integer_expected(['post_id'], data) + boolean_expected(['deleted'], data) + + post_id = data['post_id'] + deleted = data['deleted'] + + if deleted == True: + user_id, post = delete_post(post_id, SRC_API, auth) + else: + user_id, post = restore_post(post_id, SRC_API, auth) + + post_json = post_view(post=post, variant=4, user_id=user_id) + return post_json diff --git a/app/shared/post.py b/app/shared/post.py index 5116b9d8..08e5109e 100644 --- a/app/shared/post.py +++ b/app/shared/post.py @@ -421,3 +421,55 @@ def edit_post(input, post, type, src, user=None, auth=None, uploaded_file=None, elif from_scratch: return post + +# just for deletes by owner (mod deletes are classed as 'remove') +def delete_post(post_id, src, auth): + if src == SRC_API: + user_id = authorise_api_user(auth) + else: + user_id = current_user.id + + post = Post.query.filter_by(id=post_id, user_id=user_id, deleted=False).one() + if post.url: + post.calculate_cross_posts(delete_only=True) + + post.deleted = True + post.deleted_by = user_id + post.author.post_count -= 1 + post.community.post_count -= 1 + db.session.commit() + if src == SRC_WEB: + flash(_('Post deleted.')) + + task_selector('delete_post', user_id=user_id, post_id=post.id) + + if src == SRC_API: + return user_id, post + else: + return + + +def restore_post(post_id, src, auth): + if src == SRC_API: + user_id = authorise_api_user(auth) + else: + user_id = current_user.id + + post = Post.query.filter_by(id=post_id, user_id=user_id, deleted=True).one() + if post.url: + post.calculate_cross_posts() + + post.deleted = False + post.deleted_by = None + post.author.post_count -= 1 + post.community.post_count -= 1 + db.session.commit() + if src == SRC_WEB: + flash(_('Post restored.')) + + task_selector('restore_post', user_id=user_id, post_id=post.id) + + if src == SRC_API: + return user_id, post + else: + return diff --git a/app/shared/reply.py b/app/shared/reply.py index d75153ec..8e6c0560 100644 --- a/app/shared/reply.py +++ b/app/shared/reply.py @@ -233,12 +233,11 @@ def edit_reply(input, reply, post, src, auth=None): # just for deletes by owner (mod deletes are classed as 'remove') def delete_reply(reply_id, src, auth): if src == SRC_API: - reply = PostReply.query.filter_by(id=reply_id, deleted=False).one() - user_id = authorise_api_user(auth, id_match=reply.user_id) + user_id = authorise_api_user(auth) else: - reply = PostReply.query.get_or_404(reply_id) user_id = current_user.id + reply = PostReply.query.filter_by(id=reply_id, user_id=user_id, deleted=False).one() reply.deleted = True reply.deleted_by = user_id @@ -259,14 +258,11 @@ def delete_reply(reply_id, src, auth): def restore_reply(reply_id, src, auth): if src == SRC_API: - reply = PostReply.query.filter_by(id=reply_id, deleted=True).one() - user_id = authorise_api_user(auth, id_match=reply.user_id) - if reply.user_id != reply.deleted_by: - raise Exception('incorrect_login') + user_id = authorise_api_user(auth) else: - reply = PostReply.query.get_or_404(reply_id) user_id = current_user.id + reply = PostReply.query.filter_by(id=reply_id, user_id=user_id, deleted=True).one() reply.deleted = False reply.deleted_by = None diff --git a/app/shared/tasks/__init__.py b/app/shared/tasks/__init__.py index 8605a5b0..7fc2d3c9 100644 --- a/app/shared/tasks/__init__.py +++ b/app/shared/tasks/__init__.py @@ -1,7 +1,7 @@ from app.shared.tasks.follows import join_community, leave_community from app.shared.tasks.likes import vote_for_post, vote_for_reply from app.shared.tasks.notes import make_reply, edit_reply -from app.shared.tasks.deletes import delete_reply, restore_reply +from app.shared.tasks.deletes import delete_reply, restore_reply, delete_post, restore_post from app.shared.tasks.flags import report_reply from app.shared.tasks.pages import make_post, edit_post @@ -20,7 +20,9 @@ def task_selector(task_key, send_async=True, **kwargs): 'restore_reply': restore_reply, 'report_reply': report_reply, 'make_post': make_post, - 'edit_post': edit_post + 'edit_post': edit_post, + 'delete_post': delete_post, + 'restore_post': restore_post } if current_app.debug: diff --git a/app/shared/tasks/deletes.py b/app/shared/tasks/deletes.py index 6ff31c79..d82902b7 100644 --- a/app/shared/tasks/deletes.py +++ b/app/shared/tasks/deletes.py @@ -1,6 +1,6 @@ from app import celery from app.activitypub.signature import default_context, post_request -from app.models import CommunityBan, PostReply, User +from app.models import CommunityBan, Instance, Post, PostReply, User, UserFollower from app.utils import gibberish, instance_banned from flask import current_app @@ -34,10 +34,31 @@ def restore_reply(send_async, user_id, reply_id): delete_object(user_id, reply, is_restore=True) -def delete_object(user_id, object, is_restore=False): +@celery.task +def delete_post(send_async, user_id, post_id): + post = Post.query.filter_by(id=post_id).one() + delete_object(user_id, post, is_post=True) + + +@celery.task +def restore_post(send_async, user_id, post_id): + post = Post.query.filter_by(id=post_id).one() + delete_object(user_id, post, is_post=True, is_restore=True) + + +def delete_object(user_id, object, is_post=False, is_restore=False): user = User.query.filter_by(id=user_id).one() community = object.community - if community.local_only or not community.instance.online(): + + # local_only communities can also be used to send activity to User Followers (only applies to posts, not comments) + # return now though, if there aren't any + if not is_post and community.local_only: + return + followers = UserFollower.query.filter_by(local_user_id=user.id).all() + if not followers and community.local_only: + return + + if not community.instance.online(): return banned = CommunityBan.query.filter_by(user_id=user_id, community_id=community.id).first() @@ -75,6 +96,8 @@ def delete_object(user_id, object, is_restore=False): 'cc': cc } + domains_sent_to = [] + if community.is_local(): if is_restore: del undo['@context'] @@ -97,9 +120,24 @@ def delete_object(user_id, object, is_restore=False): } for instance in community.following_instances(): if instance.inbox and instance.online() and not user.has_blocked_instance(instance.id) and not instance_banned(instance.domain): - post_request(instance.inbox, announce, community.private_key, community.public_url() + '#main-key') + post_request(instance.inbox, announce, community.private_key, community.public_url() + '#main-key') + domains_sent_to.append(instance.domain) else: payload = undo if is_restore else delete post_request(community.ap_inbox_url, payload, user.private_key, user.public_url() + '#main-key') + domains_sent_to.append(community.instance.domain) + + if is_post and followers: + payload = undo if is_restore else delete + for follower in followers: + user_details = User.query.get(follower.remote_user_id) + if user_details: + payload['cc'].append(user_details.public_url()) + instances = Instance.query.join(User, User.instance_id == Instance.id).join(UserFollower, UserFollower.remote_user_id == User.id) + instances = instances.filter(UserFollower.local_user_id == user.id).filter(Instance.gone_forever == False) + for instance in instances: + if instance.domain not in domains_sent_to: + post_request(instance.inbox, payload, user.private_key, user.public_url() + '#main-key') + From dc5c0fd7b49c12af5ea050cdc6f6b0ca36a68ba1 Mon Sep 17 00:00:00 2001 From: freamon Date: Sat, 18 Jan 2025 17:25:54 +0000 Subject: [PATCH 17/19] Update resolve_remote_post_from_search() to not require 'audience' field --- app/activitypub/util.py | 166 ++++++++++++++++++---------------------- 1 file changed, 73 insertions(+), 93 deletions(-) diff --git a/app/activitypub/util.py b/app/activitypub/util.py index 06ff7843..5a53f6b2 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -2400,112 +2400,92 @@ def resolve_remote_post(uri: str, community_id: int, announce_actor=None, store_ return None +# called from UI, via 'search' option in navbar def resolve_remote_post_from_search(uri: str) -> Union[Post, None]: post = Post.query.filter_by(ap_id=uri).first() if post: return post - site = Site.query.get(1) - parsed_url = urlparse(uri) uri_domain = parsed_url.netloc actor_domain = None actor = None - post_request = get_request(uri, headers={'Accept': 'application/activity+json'}) - if post_request.status_code == 200: - post_data = post_request.json() - post_request.close() - # check again that it doesn't already exist (can happen with different but equivalent URLs) - post = Post.query.filter_by(ap_id=post_data['id']).first() - if post: - return post - # find the author of the post. Make sure their domain matches the site hosting it to mitigate impersonation attempts - if 'attributedTo' in post_data: - attributed_to = post_data['attributedTo'] - if isinstance(attributed_to, str): - actor = attributed_to - parsed_url = urlparse(actor) - actor_domain = parsed_url.netloc - elif isinstance(attributed_to, list): - for a in attributed_to: - if isinstance(a, dict) and a.get('type') == 'Person': - actor = a.get('id') - if isinstance(actor, str): # Ensure `actor` is a valid string - parsed_url = urlparse(actor) - actor_domain = parsed_url.netloc - break - elif isinstance(a, str): - actor = a + try: + object_request = get_request(uri, headers={'Accept': 'application/activity+json'}) + except httpx.HTTPError: + time.sleep(3) + try: + object_request = get_request(uri, headers={'Accept': 'application/activity+json'}) + except httpx.HTTPError: + return None + if object_request.status_code == 200: + try: + post_data = object_request.json() + except: + object_request.close() + return None + object_request.close() + elif object_request.status_code == 401: + try: + site = Site.query.get(1) + object_request = signed_get_request(uri, site.private_key, f"https://{current_app.config['SERVER_NAME']}/actor#main-key") + except httpx.HTTPError: + time.sleep(3) + try: + object_request = signed_get_request(uri, site.private_key, f"https://{current_app.config['SERVER_NAME']}/actor#main-key") + except httpx.HTTPError: + return None + try: + post_data = object_request.json() + except: + object_request.close() + return None + object_request.close() + else: + return None + + # check again that it doesn't already exist (can happen with different but equivalent URLs) + post = Post.query.filter_by(ap_id=post_data['id']).first() + if post: + return post + + # find the author of the post. Make sure their domain matches the site hosting it to mitigate impersonation attempts + if 'attributedTo' in post_data: + attributed_to = post_data['attributedTo'] + if isinstance(attributed_to, str): + actor = attributed_to + parsed_url = urlparse(actor) + actor_domain = parsed_url.netloc + elif isinstance(attributed_to, list): + for a in attributed_to: + if isinstance(a, dict) and a.get('type') == 'Person': + actor = a.get('id') + if isinstance(actor, str): # Ensure `actor` is a valid string parsed_url = urlparse(actor) actor_domain = parsed_url.netloc - break - if uri_domain != actor_domain: - return None + break + elif isinstance(a, str): + actor = a + parsed_url = urlparse(actor) + actor_domain = parsed_url.netloc + break + if uri_domain != actor_domain: + return None - # find the community the post was submitted to - community = None - if not community and post_data['type'] == 'Page': # lemmy - if 'audience' in post_data: - community_id = post_data['audience'] - community = find_actor_or_create(community_id, community_only=True) - - if not community and post_data['type'] == 'Video': # peertube - if 'attributedTo' in post_data and isinstance(post_data['attributedTo'], list): - for a in post_data['attributedTo']: - if a['type'] == 'Group': - community_id = a['id'] - community = find_actor_or_create(community_id, community_only=True) - if community: - break - - if not community: # mastodon, etc - if 'inReplyTo' not in post_data or post_data['inReplyTo'] != None: - return None - - if not community and 'to' in post_data and isinstance(post_data['to'], str): - community_id = post_data['to'].lower() - if not community_id == 'https://www.w3.org/ns/activitystreams#Public' and not community_id.endswith('/followers'): - community = Community.query.filter_by(ap_profile_id=community_id).first() - if not community and 'cc' in post_data and isinstance(post_data['cc'], str): - community_id = post_data['cc'].lower() - if not community_id == 'https://www.w3.org/ns/activitystreams#Public' and not community_id.endswith('/followers'): - community = Community.query.filter_by(ap_profile_id=community_id).first() - if not community and 'to' in post_data and isinstance(post_data['to'], list): - for t in post_data['to']: - community_id = t.lower() - if not community_id == 'https://www.w3.org/ns/activitystreams#Public' and not community_id.endswith('/followers'): - community = Community.query.filter_by(ap_profile_id=community_id).first() - if community: - break - if not community and 'cc' in post_data and isinstance(post_data['to'], list): - for c in post_data['cc']: - community_id = c.lower() - if not community_id == 'https://www.w3.org/ns/activitystreams#Public' and not community_id.endswith('/followers'): - community = Community.query.filter_by(ap_profile_id=community_id).first() - if community: - break - - if not community: - return None - - activity_log = ActivityPubLog(direction='in', activity_id=post_data['id'], activity_type='Resolve Post', result='failure') - if site.log_activitypub_json: - activity_log.activity_json = json.dumps(post_data) - db.session.add(activity_log) - user = find_actor_or_create(actor) - if user and community and post_data: - request_json = { - 'id': f"https://{uri_domain}/activities/create/gibberish(15)", - 'object': post_data - } - post = create_post(activity_log, community, request_json, user) - if post: - if 'published' in post_data: - post.posted_at=post_data['published'] - post.last_active=post_data['published'] - db.session.commit() - return post + # find the community the post was submitted to + community = find_community(post_data) + # find the post's author + user = find_actor_or_create(actor) + if user and community and post_data: + request_json = {'id': f"https://{uri_domain}/activities/create/gibberish(15)", 'object': post_data} + post = create_post(False, community, request_json, user) + if post: + if 'published' in post_data: + post.posted_at=post_data['published'] + post.last_active=post_data['published'] + db.session.commit() + return post return None From eb7af155ee15a5a629dd935c3260b15725fd495b Mon Sep 17 00:00:00 2001 From: freamon Date: Sat, 18 Jan 2025 17:56:09 +0000 Subject: [PATCH 18/19] API: post reports (creation) --- app/api/alpha/routes.py | 14 +++++++- app/api/alpha/utils/__init__.py | 2 +- app/api/alpha/utils/post.py | 21 ++++++++++-- app/api/alpha/views.py | 41 ++++++++++++++++++++++ app/shared/post.py | 60 ++++++++++++++++++++++++++++++++- app/shared/tasks/__init__.py | 5 +-- app/shared/tasks/flags.py | 8 ++++- 7 files changed, 143 insertions(+), 8 deletions(-) diff --git a/app/api/alpha/routes.py b/app/api/alpha/routes.py index 20c5be37..b6dcc8a2 100644 --- a/app/api/alpha/routes.py +++ b/app/api/alpha/routes.py @@ -1,7 +1,7 @@ from app.api.alpha import bp from app.api.alpha.utils import get_site, post_site_block, \ get_search, \ - get_post_list, get_post, post_post_like, put_post_save, put_post_subscribe, post_post, put_post, post_post_delete, \ + get_post_list, get_post, post_post_like, put_post_save, put_post_subscribe, post_post, put_post, post_post_delete, post_post_report, \ get_reply_list, post_reply_like, put_reply_save, put_reply_subscribe, post_reply, put_reply, post_reply_delete, post_reply_report, \ get_community_list, get_community, post_community_follow, post_community_block, \ get_user, post_user_block @@ -195,6 +195,18 @@ def post_alpha_post_delete(): return jsonify({"error": str(ex)}), 400 +@bp.route('/api/alpha/post/report', methods=['POST']) +def post_alpha_post_report(): + if not enable_api(): + return jsonify({'error': 'alpha api is not enabled'}) + try: + auth = request.headers.get('Authorization') + data = request.get_json(force=True) or {} + return jsonify(post_post_report(auth, data)) + except Exception as ex: + return jsonify({"error": str(ex)}), 400 + + # Reply @bp.route('/api/alpha/comment/list', methods=['GET']) def get_alpha_comment_list(): diff --git a/app/api/alpha/utils/__init__.py b/app/api/alpha/utils/__init__.py index 79428210..7f9d1d21 100644 --- a/app/api/alpha/utils/__init__.py +++ b/app/api/alpha/utils/__init__.py @@ -1,6 +1,6 @@ from app.api.alpha.utils.site import get_site, post_site_block from app.api.alpha.utils.misc import get_search -from app.api.alpha.utils.post import get_post_list, get_post, post_post_like, put_post_save, put_post_subscribe, post_post, put_post, post_post_delete +from app.api.alpha.utils.post import get_post_list, get_post, post_post_like, put_post_save, put_post_subscribe, post_post, put_post, post_post_delete, post_post_report from app.api.alpha.utils.reply import get_reply_list, post_reply_like, put_reply_save, put_reply_subscribe, post_reply, put_reply, post_reply_delete, post_reply_report from app.api.alpha.utils.community import get_community, get_community_list, post_community_follow, post_community_block from app.api.alpha.utils.user import get_user, post_user_block diff --git a/app/api/alpha/utils/post.py b/app/api/alpha/utils/post.py index 3da7dc21..32deb0ce 100644 --- a/app/api/alpha/utils/post.py +++ b/app/api/alpha/utils/post.py @@ -1,9 +1,9 @@ from app import cache -from app.api.alpha.views import post_view +from app.api.alpha.views import post_view, post_report_view from app.api.alpha.utils.validators import required, integer_expected, boolean_expected, string_expected from app.constants import POST_TYPE_ARTICLE, POST_TYPE_LINK, POST_TYPE_IMAGE, POST_TYPE_VIDEO from app.models import Post, Community, CommunityMember, utcnow -from app.shared.post import vote_for_post, bookmark_the_post, remove_the_bookmark_from_post, toggle_post_notification, make_post, edit_post, delete_post, restore_post +from app.shared.post import vote_for_post, bookmark_the_post, remove_the_bookmark_from_post, toggle_post_notification, make_post, edit_post, delete_post, restore_post, report_post from app.utils import authorise_api_user, blocked_users, blocked_communities, blocked_instances, community_ids_from_instances, is_image_url, is_video_url from datetime import timedelta @@ -222,3 +222,20 @@ def post_post_delete(auth, data): post_json = post_view(post=post, variant=4, user_id=user_id) return post_json + + + +def post_post_report(auth, data): + required(['post_id', 'reason'], data) + integer_expected(['post_id'], data) + string_expected(['reason'], data) + + post_id = data['post_id'] + reason = data['reason'] + input = {'reason': reason, 'description': '', 'report_remote': True} + + user_id, report = report_post(post_id, input, SRC_API, auth) + + post_json = post_report_view(report=report, post_id=post_id, user_id=user_id) + return post_json + diff --git a/app/api/alpha/views.py b/app/api/alpha/views.py index 3270ec94..ce2d301d 100644 --- a/app/api/alpha/views.py +++ b/app/api/alpha/views.py @@ -373,6 +373,47 @@ def reply_report_view(report, reply_id, user_id): return v1 +def post_report_view(report, post_id, user_id): + # views/post_report_view.dart - /post/report api endpoint + post_json = post_view(post=post_id, variant=2, user_id=user_id) + community_json = community_view(community=post_json['post']['community_id'], variant=1, stub=True) + + banned = db.session.execute(text('SELECT user_id FROM "community_ban" WHERE user_id = :user_id and community_id = :community_id'), {'user_id': report.reporter_id, 'community_id': community_json['id']}).scalar() + moderator = db.session.execute(text('SELECT is_moderator FROM "community_member" WHERE user_id = :user_id and community_id = :community_id'), {'user_id': report.reporter_id, 'community_id': community_json['id']}).scalar() + admin = db.session.execute(text('SELECT user_id FROM "user_role" WHERE user_id = :user_id and role_id = 4'), {'user_id': report.reporter_id}).scalar() + + creator_banned_from_community = True if banned else False + creator_is_moderator = True if moderator else False + creator_is_admin = True if admin else False + + v1 = { + 'post_report_view': { + 'post_report': { + 'id': report.id, + 'creator_id': report.reporter_id, + 'post_id': report.suspect_post_id, + 'original_post_name': post_json['post']['title'], + 'original_post_body': '', + 'reason': report.reasons, + 'resolved': report.status == 3, + 'published': report.created_at.isoformat() + 'Z' + }, + 'post': post_json['post'], + 'community': community_json, + 'creator': user_view(user=user_id, variant=1, stub=True), + 'post_creator': user_view(user=report.suspect_user_id, variant=1, stub=True), + 'counts': post_json['counts'], + 'creator_banned_from_community': creator_banned_from_community, + 'creator_is_moderator': creator_is_moderator, + 'creator_is_admin': creator_is_admin, + 'creator_blocked': False, + 'subscribed': post_json['subscribed'], + 'saved': post_json['saved'] + } + } + return v1 + + def search_view(type): v1 = { 'type_': type, diff --git a/app/shared/post.py b/app/shared/post.py index 08e5109e..f330a4c5 100644 --- a/app/shared/post.py +++ b/app/shared/post.py @@ -2,7 +2,7 @@ from app import db, cache from app.activitypub.util import make_image_sizes from app.constants import * from app.community.util import tags_from_string_old, end_poll_date -from app.models import File, Language, NotificationSubscription, Poll, PollChoice, Post, PostBookmark, PostVote, utcnow +from app.models import File, Language, Notification, NotificationSubscription, Poll, PollChoice, Post, PostBookmark, PostVote, Report, Site, User, utcnow from app.shared.tasks import task_selector from app.utils import render_template, authorise_api_user, shorten_string, gibberish, ensure_directory_exists, \ piefed_markdown_to_lemmy_markdown, markdown_to_html, remove_tracking_from_link, domain_from_url, \ @@ -473,3 +473,61 @@ def restore_post(post_id, src, auth): return user_id, post else: return + + +def report_post(post_id, input, src, auth=None): + if src == SRC_API: + post = Post.query.filter_by(id=post_id).one() + user_id = authorise_api_user(auth) + reason = input['reason'] + description = input['description'] + report_remote = input['report_remote'] + else: + post = Post.query.get_or_404(post_id) + user_id = current_user.id + reason = input.reasons_to_string(input.reasons.data) + description = input.description.data + report_remote = input.report_remote.data + + if post.reports == -1: # When a mod decides to ignore future reports, post.reports is set to -1 + if src == SRC_API: + raise Exception('already_reported') + else: + flash(_('Post has already been reported, thank you!')) + return + + report = Report(reasons=reason, description=description, type=1, reporter_id=user_id, suspect_post_id=post.id, suspect_community_id=post.community_id, + suspect_user_id=post.user_id, in_community_id=post.community_id, source_instance_id=1) + db.session.add(report) + + # Notify moderators + already_notified = set() + for mod in post.community.moderators(): + moderator = User.query.get(mod.user_id) + if moderator and moderator.is_local(): + notification = Notification(user_id=mod.user_id, title=_('A comment has been reported'), + url=f"https://{current_app.config['SERVER_NAME']}/comment/{post.id}", + author_id=user_id) + db.session.add(notification) + already_notified.add(mod.user_id) + post.reports += 1 + # todo: only notify admins for certain types of report + for admin in Site.admins(): + if admin.id not in already_notified: + notify = Notification(title='Suspicious content', url='/admin/reports', user_id=admin.id, author_id=user_id) + db.session.add(notify) + admin.unread_notifications += 1 + db.session.commit() + + # federate report to originating instance + if not post.community.is_local() and report_remote: + summary = reason + if description: + summary += ' - ' + description + + task_selector('report_post', user_id=user_id, post_id=post_id, summary=summary) + + if src == SRC_API: + return user_id, report + else: + return diff --git a/app/shared/tasks/__init__.py b/app/shared/tasks/__init__.py index 7fc2d3c9..9d5bba5a 100644 --- a/app/shared/tasks/__init__.py +++ b/app/shared/tasks/__init__.py @@ -2,7 +2,7 @@ from app.shared.tasks.follows import join_community, leave_community from app.shared.tasks.likes import vote_for_post, vote_for_reply from app.shared.tasks.notes import make_reply, edit_reply from app.shared.tasks.deletes import delete_reply, restore_reply, delete_post, restore_post -from app.shared.tasks.flags import report_reply +from app.shared.tasks.flags import report_reply, report_post from app.shared.tasks.pages import make_post, edit_post from flask import current_app @@ -22,7 +22,8 @@ def task_selector(task_key, send_async=True, **kwargs): 'make_post': make_post, 'edit_post': edit_post, 'delete_post': delete_post, - 'restore_post': restore_post + 'restore_post': restore_post, + 'report_post': report_post } if current_app.debug: diff --git a/app/shared/tasks/flags.py b/app/shared/tasks/flags.py index f461794f..9981ae99 100644 --- a/app/shared/tasks/flags.py +++ b/app/shared/tasks/flags.py @@ -1,6 +1,6 @@ from app import celery from app.activitypub.signature import default_context, post_request -from app.models import CommunityBan, PostReply, User +from app.models import CommunityBan, Post, PostReply, User from app.utils import gibberish, instance_banned from flask import current_app @@ -27,6 +27,12 @@ def report_reply(send_async, user_id, reply_id, summary): report_object(user_id, reply, summary) +@celery.task +def report_post(send_async, user_id, post_id, summary): + post = Post.query.filter_by(id=post_id).one() + report_object(user_id, post, summary) + + def report_object(user_id, object, summary): user = User.query.filter_by(id=user_id).one() community = object.community From 29d450d75a7c83dd9119971b64571694cc636ce6 Mon Sep 17 00:00:00 2001 From: freamon Date: Sat, 18 Jan 2025 22:43:58 +0000 Subject: [PATCH 19/19] Template for Banned from Community Notification to lead to #400 --- app/activitypub/util.py | 9 ++++-- app/chat/routes.py | 15 +++++++++- app/templates/chat/ban_from_mod.html | 42 ++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 4 deletions(-) create mode 100644 app/templates/chat/ban_from_mod.html diff --git a/app/activitypub/util.py b/app/activitypub/util.py index 5a53f6b2..5ba58ff0 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -1483,7 +1483,7 @@ def ban_user(blocker, blocked, community, core_activity): # Notify banned person notify = Notification(title=shorten_string('You have been banned from ' + community.title), - url=f'/notifications', user_id=blocked.id, + url=f'/chat/ban_from_mod/{blocked.id}/{community.id}', user_id=blocked.id, author_id=blocker.id) db.session.add(notify) if not current_app.debug: # user.unread_notifications += 1 hangs app if 'user' is the same person @@ -1503,7 +1503,10 @@ def ban_user(blocker, blocked, community, core_activity): def unban_user(blocker, blocked, community, core_activity): - reason = core_activity['summary'] if 'summary' in core_activity else '' + if 'object' in core_activity and 'summary' in core_activity['object']: + reason = core_activity['object']['summary'] + else: + reason = '' db.session.query(CommunityBan).filter(CommunityBan.community_id == community.id, CommunityBan.user_id == blocked.id).delete() community_membership_record = CommunityMember.query.filter_by(community_id=community.id, user_id=blocked.id).first() if community_membership_record: @@ -1513,7 +1516,7 @@ def unban_user(blocker, blocked, community, core_activity): if blocked.is_local(): # Notify unbanned person notify = Notification(title=shorten_string('You have been unbanned from ' + community.title), - url=f'/notifications', user_id=blocked.id, author_id=blocker.id) + url=f'/chat/ban_from_mod/{blocked.id}/{community.id}', user_id=blocked.id, author_id=blocker.id) db.session.add(notify) if not current_app.debug: # user.unread_notifications += 1 hangs app if 'user' is the same person blocked.unread_notifications += 1 # who pressed 'Re-submit this activity'. diff --git a/app/chat/routes.py b/app/chat/routes.py index 3ab13843..0b4dbfeb 100644 --- a/app/chat/routes.py +++ b/app/chat/routes.py @@ -6,7 +6,7 @@ from sqlalchemy import desc, or_, and_, text from app import db, celery from app.chat.forms import AddReply, ReportConversationForm from app.chat.util import send_message -from app.models import Site, User, Report, ChatMessage, Notification, InstanceBlock, Conversation, conversation_member +from app.models import Site, User, Report, ChatMessage, Notification, InstanceBlock, Conversation, conversation_member, CommunityBan, ModLog from app.user.forms import ReportUserForm from app.utils import render_template, moderating_communities, joined_communities, menu_topics from app.chat import bp @@ -103,6 +103,19 @@ def empty(): return render_template('chat/empty.html') +@bp.route('/chat/ban_from_mod//', methods=['GET']) +@login_required +def ban_from_mod(user_id, community_id): + active_ban = CommunityBan.query.filter_by(user_id=user_id, community_id=community_id).order_by(desc(CommunityBan.created_at)).first() + user_link = 'u/' + current_user.user_name + past_bans = ModLog.query.filter(ModLog.community_id == community_id, ModLog.link == user_link, or_(ModLog.action == 'ban_user', ModLog.action == 'unban_user')).order_by(desc(ModLog.created_at)) + if active_ban: + past_bans = past_bans.offset(1) + #if active_ban and len(past_bans) > 1: + #past_bans = past_bans + return render_template('chat/ban_from_mod.html', active_ban=active_ban, past_bans=past_bans) + + @bp.route('/chat//options', methods=['GET', 'POST']) @login_required def chat_options(conversation_id): diff --git a/app/templates/chat/ban_from_mod.html b/app/templates/chat/ban_from_mod.html new file mode 100644 index 00000000..186e0a8a --- /dev/null +++ b/app/templates/chat/ban_from_mod.html @@ -0,0 +1,42 @@ +{% if theme() and file_exists('app/templates/themes/' + theme() + '/base.html') %} + {% extends 'themes/' + theme() + '/base.html' %} +{% else %} + {% extends "base.html" %} +{% endif %} +{% set active_child = 'chats' %} + +{% block app_content %} +
+{% if active_ban %} +
Active Ban
+ + + + + + + + + + +
ReasonFromUntil
{{ active_ban.reason }}{{ active_ban.created_at.strftime('%Y-%m-%d') }}{{ active_ban.ban_until.strftime('%Y-%m-%d') if active_ban.ban_until else '' }}
+{% endif %} +{% if past_bans.count() > 0 %} +
Ban History
+ + + + + + + {% for past_ban in past_bans %} + + + + + + {% endfor %} +
ReasonFromType
{{ past_ban.reason }}{{ past_ban.created_at.strftime('%Y-%m-%d') }}{{ past_ban.action }}
+{% endif %} +
+{% endblock %}