Revert earlier decision to only send out HTML (pls see app/utils.py line 327)

app/activitypub/routes.py

@@ -296,6 +296,7 @@ def user_profile(actor):
                 }
             if user.about_html and main_user_name:
                 actor_data['summary'] = user.about_html
+                actor_data['source'] = {'content': user.about, 'mediaType': 'text/markdown'}
             if user.matrix_user_id and main_user_name:
                 actor_data['matrixUserId'] = user.matrix_user_id
             resp = jsonify(actor_data)
@@ -368,6 +369,7 @@ def community_profile(actor):
             }
             if community.description_html:
                 actor_data["summary"] = community.description_html
+                actor_data['source'] = {'content': community.description, 'mediaType': 'text/markdown'}
             if community.icon_id is not None:
                 actor_data["icon"] = {
                     "type": "Image",

app/activitypub/util.py

@@ -135,6 +135,7 @@ def post_to_page(post: Post):
         "cc": [],
         "content": post.body_html if post.body_html else '',
         "mediaType": "text/html",
+        "source": {"content": post.body if post.body else '', "mediaType": "text/markdown"},
         "attachment": [],
         "commentsEnabled": post.comments_enabled,
         "sensitive": post.nsfw or post.nsfl,
@@ -206,6 +207,7 @@ def comment_model_to_json(reply: PostReply) -> dict:
         ],
         'content': reply.body_html,
         'mediaType': 'text/html',
+        'source': {'content': reply.body, 'mediaType': 'text/markdown'},
         'published': ap_datetime(reply.created_at),
         'distinguished': False,
         'audience': reply.community.public_url(),

app/community/routes.py

@@ -38,7 +38,7 @@ from app.utils import get_setting, render_template, allowlist_html, markdown_to_
     joined_communities, moderating_communities, blocked_domains, mimetype_from_url, blocked_instances, \
     community_moderators, communities_banned_from, show_ban_message, recently_upvoted_posts, recently_downvoted_posts, \
     blocked_users, post_ranking, languages_for_form, english_language_id, menu_topics, add_to_modlog, \
-    blocked_communities, remove_tracking_from_link
+    blocked_communities, remove_tracking_from_link, piefed_markdown_to_lemmy_markdown
 from feedgen.feed import FeedGenerator
 from datetime import timezone, timedelta
 from copy import copy
@@ -60,7 +60,7 @@ def add_local():
             form.url.data = form.url.data[3:]
         form.url.data = slugify(form.url.data.strip(), separator='_').lower()
         private_key, public_key = RsaKeys.generate_keypair()
-        community = Community(title=form.community_name.data, name=form.url.data, description=form.description.data,
+        community = Community(title=form.community_name.data, name=form.url.data, description=piefed_markdown_to_lemmy_markdown(form.description.data),
                               rules=form.rules.data, nsfw=form.nsfw.data, private_key=private_key,
                               public_key=public_key, description_html=markdown_to_html(form.description.data),
                               rules_html=markdown_to_html(form.rules.data), local_only=form.local_only.data,
@@ -697,6 +697,7 @@ def federate_post(community, post):
         'cc': [],
         'content': post.body_html if post.body_html else '',
         'mediaType': 'text/html',
+        'source': {'content': post.body if post.body else '', 'mediaType': 'text/markdown'},
         'attachment': [],
         'commentsEnabled': post.comments_enabled,
         'sensitive': post.nsfw,
@@ -918,7 +919,7 @@ def community_edit(community_id: int):
         form.languages.choices = languages_for_form()
         if form.validate_on_submit():
             community.title = form.title.data
-            community.description = form.description.data
+            community.description = piefed_markdown_to_lemmy_markdown(form.description.data)
             community.description_html = markdown_to_html(form.description.data, anchors_new_tab=False)
             community.rules = form.rules.data
             community.rules_html = markdown_to_html(form.rules.data, anchors_new_tab=False)

app/community/util.py

@@ -19,7 +19,8 @@ from app.models import Community, File, BannedInstances, PostReply, PostVote, Po
     Instance, Notification, User, ActivityPubLog, NotificationSubscription, Language, Tag, PollChoice, Poll
 from app.utils import get_request, gibberish, markdown_to_html, domain_from_url, allowlist_html, \
     is_image_url, ensure_directory_exists, inbox_domain, post_ranking, shorten_string, parse_page, \
-    remove_tracking_from_link, ap_datetime, instance_banned, blocked_phrases, url_to_thumbnail_file, opengraph_parse
+    remove_tracking_from_link, ap_datetime, instance_banned, blocked_phrases, url_to_thumbnail_file, opengraph_parse, \
+    piefed_markdown_to_lemmy_markdown
 from sqlalchemy import func, desc, text
 import os
 
@@ -254,7 +255,7 @@ def save_post(form, post: Post, type: int):
     post.language_id = form.language_id.data
     current_user.language_id = form.language_id.data
     post.title = form.title.data
-    post.body = form.body.data
+    post.body = piefed_markdown_to_lemmy_markdown(form.body.data)
     post.body_html = markdown_to_html(post.body)
     if not type or type == POST_TYPE_ARTICLE:
         post.type = POST_TYPE_ARTICLE

app/post/routes.py

@@ -31,7 +31,7 @@ from app.utils import get_setting, render_template, allowlist_html, markdown_to_
     reply_already_exists, reply_is_just_link_to_gif_reaction, confidence, moderating_communities, joined_communities, \
     blocked_instances, blocked_domains, community_moderators, blocked_phrases, show_ban_message, recently_upvoted_posts, \
     recently_downvoted_posts, recently_upvoted_post_replies, recently_downvoted_post_replies, reply_is_stupid, \
-    languages_for_form, menu_topics, add_to_modlog, blocked_communities
+    languages_for_form, menu_topics, add_to_modlog, blocked_communities, piefed_markdown_to_lemmy_markdown
 
 
 def show_post(post_id: int):
@@ -103,7 +103,7 @@ def show_post(post_id: int):
                 flash(_('You have already upvoted the post, you do not need to say "this" also.'), 'error')
             return redirect(url_for('activitypub.post_ap', post_id=post_id))
 
-        reply = PostReply(user_id=current_user.id, post_id=post.id, community_id=community.id, body=form.body.data,
+        reply = PostReply(user_id=current_user.id, post_id=post.id, community_id=community.id, body=piefed_markdown_to_lemmy_markdown(form.body.data),
                           body_html=markdown_to_html(form.body.data), body_html_safe=True,
                           from_bot=current_user.bot, nsfw=post.nsfw, nsfl=post.nsfl,
                           notify_author=form.notify_author.data, language_id=form.language_id.data, instance_id=1)
@@ -162,6 +162,7 @@ def show_post(post_id: int):
             'content': reply.body_html,
             'inReplyTo': post.profile_id(),
             'mediaType': 'text/html',
+            'source': {'content': reply.body, 'mediaType': 'text/markdown'},
             'published': ap_datetime(utcnow()),
             'distinguished': False,
             'audience': community.public_url(),
@@ -594,7 +595,7 @@ def add_reply(post_id: int, comment_id: int):
         current_user.ip_address = ip_address()
         current_user.language_id = form.language_id.data
         reply = PostReply(user_id=current_user.id, post_id=post.id, parent_id=in_reply_to.id, depth=in_reply_to.depth + 1,
-                          community_id=post.community.id, body=form.body.data,
+                          community_id=post.community.id, body=piefed_markdown_to_lemmy_markdown(form.body.data),
                           body_html=markdown_to_html(form.body.data), body_html_safe=True,
                           from_bot=current_user.bot, nsfw=post.nsfw, nsfl=post.nsfl,
                           notify_author=form.notify_author.data, instance_id=1, language_id=form.language_id.data)
@@ -656,6 +657,7 @@ def add_reply(post_id: int, comment_id: int):
                 'inReplyTo': in_reply_to.profile_id(),
                 'url': reply.profile_id(),
                 'mediaType': 'text/html',
+                'source': {'content': reply.body, 'mediaType': 'text/markdown'},
                 'published': ap_datetime(utcnow()),
                 'distinguished': False,
                 'audience': post.community.public_url(),
@@ -911,6 +913,7 @@ def federate_post_update(post):
         'cc': [],
         'content': post.body_html if post.body_html else '',
         'mediaType': 'text/html',
+        'source': {'content': post.body if post.body else '', 'mediaType': 'text/markdown'},
         'attachment': [],
         'commentsEnabled': post.comments_enabled,
         'sensitive': post.nsfw,
@@ -1015,6 +1018,7 @@ def federate_post_edit_to_user_followers(post):
         ],
         'content': '',
         'mediaType': 'text/html',
+        'source': {'content': post.body if post.body else '', 'mediaType': 'text/markdown'},
         'attachment': [],
         'commentsEnabled': post.comments_enabled,
         'sensitive': post.nsfw,
@@ -1545,7 +1549,7 @@ def post_reply_edit(post_id: int, comment_id: int):
     form.language_id.choices = languages_for_form()
     if post_reply.user_id == current_user.id or post.community.is_moderator():
         if form.validate_on_submit():
-            post_reply.body = form.body.data
+            post_reply.body = piefed_markdown_to_lemmy_markdown(form.body.data)
             post_reply.body_html = markdown_to_html(form.body.data)
             post_reply.notify_author = form.notify_author.data
             post.community.last_active = utcnow()
@@ -1575,6 +1579,7 @@ def post_reply_edit(post_id: int, comment_id: int):
                     'inReplyTo': in_reply_to.profile_id(),
                     'url': post_reply.public_url(),
                     'mediaType': 'text/html',
+                    'source': {'content': post_reply.body, 'mediaType': 'text/markdown'},
                     'published': ap_datetime(post_reply.posted_at),
                     'updated': ap_datetime(post_reply.edited_at),
                     'distinguished': False,

app/user/routes.py

@@ -26,7 +26,7 @@ from app.utils import get_setting, render_template, markdown_to_html, user_acces
     is_image_url, ensure_directory_exists, gibberish, file_get_contents, community_membership, user_filters_home, \
     user_filters_posts, user_filters_replies, moderating_communities, joined_communities, theme_list, blocked_instances, \
     allowlist_html, recently_upvoted_posts, recently_downvoted_posts, blocked_users, menu_topics, add_to_modlog, \
-    blocked_communities
+    blocked_communities, piefed_markdown_to_lemmy_markdown
 from sqlalchemy import desc, or_, text
 import os
 import json as python_json
@@ -130,7 +130,7 @@ def edit_profile(actor):
         current_user.email = form.email.data.strip()
         if form.password_field.data.strip() != '':
             current_user.set_password(form.password_field.data)
-        current_user.about = form.about.data
+        current_user.about = piefed_markdown_to_lemmy_markdown(form.about.data)
         current_user.about_html = markdown_to_html(form.about.data)
         current_user.matrix_user_id = form.matrixuserid.data
         current_user.bot = form.bot.data

app/utils.py

@@ -321,6 +321,20 @@ def markdown_to_html(markdown_text, anchors_new_tab=True) -> str:
         return ''
 
 
+# this function lets local users use the more intuitive soft-breaks for newlines, but actually stores the Markdown in Lemmy-compatible format
+# Reasons for this:
+# 1. it's what any adapted Lemmy apps using an API would expect
+# 2. we need to revert to sending out Markdown in 'source' because:
+#    a. Lemmy doesn't convert '<details><summary>' back into its '::: spoiler' format
+#    b. anything coming from another PieFed instance would get reduced with html_to_text()
+#    c. raw 'https' strings in code blocks are being converted into <a> links for HTML that Lemmy then converts back into []()
+def piefed_markdown_to_lemmy_markdown(piefed_markdown: str):
+    # only difference is newlines for soft breaks.
+    re_breaks = re.compile(r'(\S)(\r\n)')
+    lemmy_markdown = re_breaks.sub(r'\1  \2', piefed_markdown)
+    return lemmy_markdown
+
+
 def markdown_to_text(markdown_text) -> str:
     if not markdown_text or markdown_text == '':
         return ''