From 6413490cc34db6580d4e1557f5ccd7c91e91afb3 Mon Sep 17 00:00:00 2001 From: freamon Date: Sun, 22 Sep 2024 13:42:02 +0000 Subject: [PATCH] Revert earlier decision to only send out HTML (pls see app/utils.py line 327) --- app/activitypub/routes.py | 2 ++ app/activitypub/util.py | 2 ++ app/community/routes.py | 7 ++++--- app/community/util.py | 5 +++-- app/post/routes.py | 13 +++++++++---- app/user/routes.py | 4 ++-- app/utils.py | 14 ++++++++++++++ 7 files changed, 36 insertions(+), 11 deletions(-) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index fe8c1855..11382909 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -296,6 +296,7 @@ def user_profile(actor): } if user.about_html and main_user_name: actor_data['summary'] = user.about_html + actor_data['source'] = {'content': user.about, 'mediaType': 'text/markdown'} if user.matrix_user_id and main_user_name: actor_data['matrixUserId'] = user.matrix_user_id resp = jsonify(actor_data) @@ -368,6 +369,7 @@ def community_profile(actor): } if community.description_html: actor_data["summary"] = community.description_html + actor_data['source'] = {'content': community.description, 'mediaType': 'text/markdown'} if community.icon_id is not None: actor_data["icon"] = { "type": "Image", diff --git a/app/activitypub/util.py b/app/activitypub/util.py index 4cf39aa4..e77c81bc 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -135,6 +135,7 @@ def post_to_page(post: Post): "cc": [], "content": post.body_html if post.body_html else '', "mediaType": "text/html", + "source": {"content": post.body if post.body else '', "mediaType": "text/markdown"}, "attachment": [], "commentsEnabled": post.comments_enabled, "sensitive": post.nsfw or post.nsfl, @@ -206,6 +207,7 @@ def comment_model_to_json(reply: PostReply) -> dict: ], 'content': reply.body_html, 'mediaType': 'text/html', + 'source': {'content': reply.body, 'mediaType': 'text/markdown'}, 'published': ap_datetime(reply.created_at), 'distinguished': False, 'audience': reply.community.public_url(), diff --git a/app/community/routes.py b/app/community/routes.py index 45ba3295..0e0e1556 100644 --- a/app/community/routes.py +++ b/app/community/routes.py @@ -38,7 +38,7 @@ from app.utils import get_setting, render_template, allowlist_html, markdown_to_ joined_communities, moderating_communities, blocked_domains, mimetype_from_url, blocked_instances, \ community_moderators, communities_banned_from, show_ban_message, recently_upvoted_posts, recently_downvoted_posts, \ blocked_users, post_ranking, languages_for_form, english_language_id, menu_topics, add_to_modlog, \ - blocked_communities, remove_tracking_from_link + blocked_communities, remove_tracking_from_link, piefed_markdown_to_lemmy_markdown from feedgen.feed import FeedGenerator from datetime import timezone, timedelta from copy import copy @@ -60,7 +60,7 @@ def add_local(): form.url.data = form.url.data[3:] form.url.data = slugify(form.url.data.strip(), separator='_').lower() private_key, public_key = RsaKeys.generate_keypair() - community = Community(title=form.community_name.data, name=form.url.data, description=form.description.data, + community = Community(title=form.community_name.data, name=form.url.data, description=piefed_markdown_to_lemmy_markdown(form.description.data), rules=form.rules.data, nsfw=form.nsfw.data, private_key=private_key, public_key=public_key, description_html=markdown_to_html(form.description.data), rules_html=markdown_to_html(form.rules.data), local_only=form.local_only.data, @@ -697,6 +697,7 @@ def federate_post(community, post): 'cc': [], 'content': post.body_html if post.body_html else '', 'mediaType': 'text/html', + 'source': {'content': post.body if post.body else '', 'mediaType': 'text/markdown'}, 'attachment': [], 'commentsEnabled': post.comments_enabled, 'sensitive': post.nsfw, @@ -918,7 +919,7 @@ def community_edit(community_id: int): form.languages.choices = languages_for_form() if form.validate_on_submit(): community.title = form.title.data - community.description = form.description.data + community.description = piefed_markdown_to_lemmy_markdown(form.description.data) community.description_html = markdown_to_html(form.description.data, anchors_new_tab=False) community.rules = form.rules.data community.rules_html = markdown_to_html(form.rules.data, anchors_new_tab=False) diff --git a/app/community/util.py b/app/community/util.py index f926cc9d..fbc9e83e 100644 --- a/app/community/util.py +++ b/app/community/util.py @@ -19,7 +19,8 @@ from app.models import Community, File, BannedInstances, PostReply, PostVote, Po Instance, Notification, User, ActivityPubLog, NotificationSubscription, Language, Tag, PollChoice, Poll from app.utils import get_request, gibberish, markdown_to_html, domain_from_url, allowlist_html, \ is_image_url, ensure_directory_exists, inbox_domain, post_ranking, shorten_string, parse_page, \ - remove_tracking_from_link, ap_datetime, instance_banned, blocked_phrases, url_to_thumbnail_file, opengraph_parse + remove_tracking_from_link, ap_datetime, instance_banned, blocked_phrases, url_to_thumbnail_file, opengraph_parse, \ + piefed_markdown_to_lemmy_markdown from sqlalchemy import func, desc, text import os @@ -254,7 +255,7 @@ def save_post(form, post: Post, type: int): post.language_id = form.language_id.data current_user.language_id = form.language_id.data post.title = form.title.data - post.body = form.body.data + post.body = piefed_markdown_to_lemmy_markdown(form.body.data) post.body_html = markdown_to_html(post.body) if not type or type == POST_TYPE_ARTICLE: post.type = POST_TYPE_ARTICLE diff --git a/app/post/routes.py b/app/post/routes.py index d43b449d..452a1f27 100644 --- a/app/post/routes.py +++ b/app/post/routes.py @@ -31,7 +31,7 @@ from app.utils import get_setting, render_template, allowlist_html, markdown_to_ reply_already_exists, reply_is_just_link_to_gif_reaction, confidence, moderating_communities, joined_communities, \ blocked_instances, blocked_domains, community_moderators, blocked_phrases, show_ban_message, recently_upvoted_posts, \ recently_downvoted_posts, recently_upvoted_post_replies, recently_downvoted_post_replies, reply_is_stupid, \ - languages_for_form, menu_topics, add_to_modlog, blocked_communities + languages_for_form, menu_topics, add_to_modlog, blocked_communities, piefed_markdown_to_lemmy_markdown def show_post(post_id: int): @@ -103,7 +103,7 @@ def show_post(post_id: int): flash(_('You have already upvoted the post, you do not need to say "this" also.'), 'error') return redirect(url_for('activitypub.post_ap', post_id=post_id)) - reply = PostReply(user_id=current_user.id, post_id=post.id, community_id=community.id, body=form.body.data, + reply = PostReply(user_id=current_user.id, post_id=post.id, community_id=community.id, body=piefed_markdown_to_lemmy_markdown(form.body.data), body_html=markdown_to_html(form.body.data), body_html_safe=True, from_bot=current_user.bot, nsfw=post.nsfw, nsfl=post.nsfl, notify_author=form.notify_author.data, language_id=form.language_id.data, instance_id=1) @@ -162,6 +162,7 @@ def show_post(post_id: int): 'content': reply.body_html, 'inReplyTo': post.profile_id(), 'mediaType': 'text/html', + 'source': {'content': reply.body, 'mediaType': 'text/markdown'}, 'published': ap_datetime(utcnow()), 'distinguished': False, 'audience': community.public_url(), @@ -594,7 +595,7 @@ def add_reply(post_id: int, comment_id: int): current_user.ip_address = ip_address() current_user.language_id = form.language_id.data reply = PostReply(user_id=current_user.id, post_id=post.id, parent_id=in_reply_to.id, depth=in_reply_to.depth + 1, - community_id=post.community.id, body=form.body.data, + community_id=post.community.id, body=piefed_markdown_to_lemmy_markdown(form.body.data), body_html=markdown_to_html(form.body.data), body_html_safe=True, from_bot=current_user.bot, nsfw=post.nsfw, nsfl=post.nsfl, notify_author=form.notify_author.data, instance_id=1, language_id=form.language_id.data) @@ -656,6 +657,7 @@ def add_reply(post_id: int, comment_id: int): 'inReplyTo': in_reply_to.profile_id(), 'url': reply.profile_id(), 'mediaType': 'text/html', + 'source': {'content': reply.body, 'mediaType': 'text/markdown'}, 'published': ap_datetime(utcnow()), 'distinguished': False, 'audience': post.community.public_url(), @@ -911,6 +913,7 @@ def federate_post_update(post): 'cc': [], 'content': post.body_html if post.body_html else '', 'mediaType': 'text/html', + 'source': {'content': post.body if post.body else '', 'mediaType': 'text/markdown'}, 'attachment': [], 'commentsEnabled': post.comments_enabled, 'sensitive': post.nsfw, @@ -1015,6 +1018,7 @@ def federate_post_edit_to_user_followers(post): ], 'content': '', 'mediaType': 'text/html', + 'source': {'content': post.body if post.body else '', 'mediaType': 'text/markdown'}, 'attachment': [], 'commentsEnabled': post.comments_enabled, 'sensitive': post.nsfw, @@ -1545,7 +1549,7 @@ def post_reply_edit(post_id: int, comment_id: int): form.language_id.choices = languages_for_form() if post_reply.user_id == current_user.id or post.community.is_moderator(): if form.validate_on_submit(): - post_reply.body = form.body.data + post_reply.body = piefed_markdown_to_lemmy_markdown(form.body.data) post_reply.body_html = markdown_to_html(form.body.data) post_reply.notify_author = form.notify_author.data post.community.last_active = utcnow() @@ -1575,6 +1579,7 @@ def post_reply_edit(post_id: int, comment_id: int): 'inReplyTo': in_reply_to.profile_id(), 'url': post_reply.public_url(), 'mediaType': 'text/html', + 'source': {'content': post_reply.body, 'mediaType': 'text/markdown'}, 'published': ap_datetime(post_reply.posted_at), 'updated': ap_datetime(post_reply.edited_at), 'distinguished': False, diff --git a/app/user/routes.py b/app/user/routes.py index 0aa16a31..6d40d3cf 100644 --- a/app/user/routes.py +++ b/app/user/routes.py @@ -26,7 +26,7 @@ from app.utils import get_setting, render_template, markdown_to_html, user_acces is_image_url, ensure_directory_exists, gibberish, file_get_contents, community_membership, user_filters_home, \ user_filters_posts, user_filters_replies, moderating_communities, joined_communities, theme_list, blocked_instances, \ allowlist_html, recently_upvoted_posts, recently_downvoted_posts, blocked_users, menu_topics, add_to_modlog, \ - blocked_communities + blocked_communities, piefed_markdown_to_lemmy_markdown from sqlalchemy import desc, or_, text import os import json as python_json @@ -130,7 +130,7 @@ def edit_profile(actor): current_user.email = form.email.data.strip() if form.password_field.data.strip() != '': current_user.set_password(form.password_field.data) - current_user.about = form.about.data + current_user.about = piefed_markdown_to_lemmy_markdown(form.about.data) current_user.about_html = markdown_to_html(form.about.data) current_user.matrix_user_id = form.matrixuserid.data current_user.bot = form.bot.data diff --git a/app/utils.py b/app/utils.py index 5ddc8819..207c063f 100644 --- a/app/utils.py +++ b/app/utils.py @@ -321,6 +321,20 @@ def markdown_to_html(markdown_text, anchors_new_tab=True) -> str: return '' +# this function lets local users use the more intuitive soft-breaks for newlines, but actually stores the Markdown in Lemmy-compatible format +# Reasons for this: +# 1. it's what any adapted Lemmy apps using an API would expect +# 2. we need to revert to sending out Markdown in 'source' because: +# a. Lemmy doesn't convert '
' back into its '::: spoiler' format +# b. anything coming from another PieFed instance would get reduced with html_to_text() +# c. raw 'https' strings in code blocks are being converted into links for HTML that Lemmy then converts back into []() +def piefed_markdown_to_lemmy_markdown(piefed_markdown: str): + # only difference is newlines for soft breaks. + re_breaks = re.compile(r'(\S)(\r\n)') + lemmy_markdown = re_breaks.sub(r'\1 \2', piefed_markdown) + return lemmy_markdown + + def markdown_to_text(markdown_text) -> str: if not markdown_text or markdown_text == '': return ''