Merge remote-tracking branch 'origin/main'

app/activitypub/routes.py

@@ -25,7 +25,7 @@ from app.activitypub.util import public_key, users_total, active_half_year, acti
 from app.utils import gibberish, get_setting, is_image_url, allowlist_html, render_template, \
     domain_from_url, markdown_to_html, community_membership, ap_datetime, ip_address, can_downvote, \
     can_upvote, can_create_post, awaken_dormant_instance, shorten_string, can_create_post_reply, sha256_digest, \
-    community_moderators
+    community_moderators, lemmy_markdown_to_html
 import werkzeug.exceptions
 
 
@@ -282,12 +282,8 @@ def user_profile(actor):
                     "type": "Image",
                     "url": f"https://{current_app.config['SERVER_NAME']}{user.cover_image()}"
                 }
-            if user.about:
-                actor_data['source'] = {
-                    "content": user.about,
-                    "mediaType": "text/markdown"
-                }
-                actor_data['summary'] = markdown_to_html(user.about)
+            if user.about_html:
+                actor_data['summary'] = user.about_html
             if user.matrix_user_id:
                 actor_data['matrixUserId'] = user.matrix_user_id
             resp = jsonify(actor_data)
@@ -332,7 +328,6 @@ def community_profile(actor):
                 "type": "Group",
                 "id": f"https://{server}/c/{actor}",
                 "name": community.title,
-                "summary": community.description,
                 "sensitive": True if community.nsfw or community.nsfl else False,
                 "preferredUsername": actor,
                 "inbox": f"https://{server}/c/{actor}/inbox",
@@ -356,6 +351,8 @@ def community_profile(actor):
                 "published": ap_datetime(community.created_at),
                 "updated": ap_datetime(community.last_active),
             }
+            if community.description_html:
+                actor_data["summary"] = community.description_html
             if community.icon_id is not None:
                 actor_data["icon"] = {
                     "type": "Image",
@@ -477,7 +474,7 @@ def process_inbox_request(request_json, activitypublog_id, ip_address):
                                 encrypted = request_json['object']['encrypted'] if 'encrypted' in request_json['object'] else None
                                 new_message = ChatMessage(sender_id=sender.id, recipient_id=recipient.id, conversation_id=existing_conversation.id,
                                                           body=request_json['object']['source']['content'],
-                                                          body_html=allowlist_html(markdown_to_html(request_json['object']['source']['content'])),
+                                                          body_html=lemmy_markdown_to_html(request_json['object']['source']['content']),
                                                           encrypted=encrypted)
                                 db.session.add(new_message)
                                 existing_conversation.updated_at = utcnow()

app/activitypub/util.py

@@ -30,7 +30,7 @@ from app.utils import get_request, allowlist_html, get_setting, ap_datetime, mar
     is_image_url, domain_from_url, gibberish, ensure_directory_exists, markdown_to_text, head_request, post_ranking, \
     shorten_string, reply_already_exists, reply_is_just_link_to_gif_reaction, confidence, remove_tracking_from_link, \
     blocked_phrases, microblog_content_to_title, generate_image_from_video_url, is_video_url, reply_is_stupid, \
-    notification_subscribers, communities_banned_from
+    notification_subscribers, communities_banned_from, lemmy_markdown_to_html
 
 
 def public_key():
@@ -148,10 +148,6 @@ def post_to_activity(post: Post, community: Community):
                 "cc": [],
                 "content": post.body_html if post.body_html else '',
                 "mediaType": "text/html",
-                "source": {
-                    "content": post.body if post.body else '',
-                    "mediaType": "text/markdown"
-                },
                 "attachment": [],
                 "commentsEnabled": post.comments_enabled,
                 "sensitive": post.nsfw or post.nsfl,
@@ -200,10 +196,6 @@ def post_to_page(post: Post, community: Community):
         "cc": [],
         "content": post.body_html if post.body_html else '',
         "mediaType": "text/html",
-        "source": {
-            "content": post.body if post.body else '',
-            "mediaType": "text/markdown"
-        },
         "attachment": [],
         "commentsEnabled": post.comments_enabled,
         "sensitive": post.nsfw or post.nsfl,
@@ -531,7 +523,7 @@ def refresh_community_profile_task(community_id):
             community.title = activity_json['name']
             community.description = activity_json['summary'] if 'summary' in activity_json else ''
             community.rules = activity_json['rules'] if 'rules' in activity_json else ''
-            community.rules_html = markdown_to_html(activity_json['rules'] if 'rules' in activity_json else '')
+            community.rules_html = lemmy_markdown_to_html(activity_json['rules'] if 'rules' in activity_json else '')
             community.restricted_to_mods = activity_json['postingRestrictedToMods']
             community.new_mods_wanted = activity_json['newModsWanted'] if 'newModsWanted' in activity_json else False
             community.private_mods = activity_json['privateMods'] if 'privateMods' in activity_json else False
@@ -542,7 +534,7 @@ def refresh_community_profile_task(community_id):
             if 'source' in activity_json and \
                     activity_json['source']['mediaType'] == 'text/markdown':
                 community.description = activity_json['source']['content']
-                community.description_html = markdown_to_html(community.description)
+                community.description_html = lemmy_markdown_to_html(community.description)
             elif 'content' in activity_json:
                 community.description_html = allowlist_html(activity_json['content'])
                 community.description = ''
@@ -672,7 +664,7 @@ def actor_json_to_model(activity_json, address, server):
                               title=activity_json['name'],
                               description=activity_json['summary'] if 'summary' in activity_json else '',
                               rules=activity_json['rules'] if 'rules' in activity_json else '',
-                              rules_html=markdown_to_html(activity_json['rules'] if 'rules' in activity_json else ''),
+                              rules_html=lemmy_markdown_to_html(activity_json['rules'] if 'rules' in activity_json else ''),
                               nsfw=activity_json['sensitive'],
                               restricted_to_mods=activity_json['postingRestrictedToMods'],
                               new_mods_wanted=activity_json['newModsWanted'] if 'newModsWanted' in activity_json else False,
@@ -698,7 +690,7 @@ def actor_json_to_model(activity_json, address, server):
         if 'source' in activity_json and \
                 activity_json['source']['mediaType'] == 'text/markdown':
             community.description = activity_json['source']['content']
-            community.description_html = markdown_to_html(community.description)
+            community.description_html = lemmy_markdown_to_html(community.description)
         elif 'content' in activity_json:
             community.description_html = allowlist_html(activity_json['content'])
             community.description = ''
@@ -741,7 +733,7 @@ def post_json_to_model(activity_log, post_json, user, community) -> Post:
         if 'source' in post_json and \
                 post_json['source']['mediaType'] == 'text/markdown':
             post.body = post_json['source']['content']
-            post.body_html = markdown_to_html(post.body)
+            post.body_html = lemmy_markdown_to_html(post.body)
         elif 'content' in post_json:
             post.body_html = allowlist_html(post_json['content'])
             post.body = ''
@@ -947,7 +939,7 @@ def parse_summary(user_json) -> str:
     if 'source' in user_json and user_json['source'].get('mediaType') == 'text/markdown':
         # Convert Markdown to HTML
         markdown_text = user_json['source']['content']
-        html_content = allowlist_html(markdown_to_html(markdown_text))
+        html_content = lemmy_markdown_to_html(markdown_text)
         return html_content
     elif 'summary' in user_json:
         return allowlist_html(user_json['summary'])
@@ -1308,7 +1300,7 @@ def delete_post_or_comment_task(user_ap_id, community_ap_id, to_be_deleted_ap_id
                 to_delete.post.reply_count -= 1
                 if to_delete.has_replies():
                     to_delete.body = 'Deleted by author' if to_delete.author.id == deletor.id else 'Deleted by moderator'
-                    to_delete.body_html = markdown_to_html(to_delete.body)
+                    to_delete.body_html = lemmy_markdown_to_html(to_delete.body)
                 else:
                     to_delete.delete_dependencies()
                     db.session.delete(to_delete)
@@ -1348,7 +1340,7 @@ def create_post_reply(activity_log: ActivityPubLog, community: Community, in_rep
                 'mediaType' in request_json['object']['source'] and \
                 request_json['object']['source']['mediaType'] == 'text/markdown':
             post_reply.body = request_json['object']['source']['content']
-            post_reply.body_html = markdown_to_html(post_reply.body)
+            post_reply.body_html = lemmy_markdown_to_html(post_reply.body)
         elif 'content' in request_json['object']:   # Kbin
             post_reply.body_html = allowlist_html(request_json['object']['content'])
             post_reply.body = ''
@@ -1371,7 +1363,7 @@ def create_post_reply(activity_log: ActivityPubLog, community: Community, in_rep
                     post.body = "🤖 I'm a bot that provides automatic summaries for articles:\n::: spoiler Click here to see the summary\n" + post_reply.body + '\n:::'
                 else:
                     post.body = post_reply.body
-                post.body_html = allowlist_html(markdown_to_html(post.body) + '\n\n<small><span class="render_username">Generated using AI by: <a href="/u/autotldr@lemmings.world" title="AutoTL;DR">AutoTL;DR</a></span></small>')
+                post.body_html = lemmy_markdown_to_html(post.body) + '\n\n<small><span class="render_username">Generated using AI by: <a href="/u/autotldr@lemmings.world" title="AutoTL;DR">AutoTL;DR</a></span></small>'
                 db.session.commit()
                 return None
 
@@ -1468,7 +1460,7 @@ def create_post(activity_log: ActivityPubLog, community: Community, request_json
     # Get post content. Lemmy and Kbin put this in different places.
     if 'source' in request_json['object'] and isinstance(request_json['object']['source'], dict) and request_json['object']['source']['mediaType'] == 'text/markdown': # Lemmy
         post.body = request_json['object']['source']['content']
-        post.body_html = markdown_to_html(post.body)
+        post.body_html = lemmy_markdown_to_html(post.body)
     elif 'content' in request_json['object'] and request_json['object']['content'] is not None: # Kbin
         post.body_html = allowlist_html(request_json['object']['content'])
         post.body = ''
@@ -1639,7 +1631,7 @@ def update_post_reply_from_activity(reply: PostReply, request_json: dict):
             isinstance(request_json['object']['source'], dict) and \
             request_json['object']['source']['mediaType'] == 'text/markdown':
         reply.body = request_json['object']['source']['content']
-        reply.body_html = markdown_to_html(reply.body)
+        reply.body_html = lemmy_markdown_to_html(reply.body)
     elif 'content' in request_json['object']:
         reply.body_html = allowlist_html(request_json['object']['content'])
         reply.body = ''
@@ -1663,7 +1655,7 @@ def update_post_from_activity(post: Post, request_json: dict):
             isinstance(request_json['object']['source'], dict) and \
             request_json['object']['source']['mediaType'] == 'text/markdown':
         post.body = request_json['object']['source']['content']
-        post.body_html = markdown_to_html(post.body)
+        post.body_html = lemmy_markdown_to_html(post.body)
     elif 'content' in request_json['object'] and request_json['object']['content'] is not None: # Kbin
         post.body_html = allowlist_html(request_json['object']['content'])
         post.body = ''

app/chat/util.py

@@ -12,7 +12,7 @@ from app.utils import allowlist_html, shorten_string, gibberish, markdown_to_htm
 def send_message(message: str, conversation_id: int) -> ChatMessage:
     conversation = Conversation.query.get(conversation_id)
     reply = ChatMessage(sender_id=current_user.id, conversation_id=conversation.id,
-                        body=message, body_html=allowlist_html(markdown_to_html(message)))
+                        body=message, body_html=markdown_to_html(message))
     conversation.updated_at = utcnow()
     db.session.add(reply)
     db.session.commit()

app/post/routes.py

@@ -155,10 +155,6 @@ def show_post(post_id: int):
             'content': reply.body_html,
             'inReplyTo': post.profile_id(),
             'mediaType': 'text/html',
-            'source': {
-                'content': reply.body,
-                'mediaType': 'text/markdown'
-            },
             'published': ap_datetime(utcnow()),
             'distinguished': False,
             'audience': community.public_url(),
@@ -689,10 +685,6 @@ def add_reply(post_id: int, comment_id: int):
                 'inReplyTo': in_reply_to.profile_id(),
                 'url': reply.profile_id(),
                 'mediaType': 'text/html',
-                'source': {
-                    'content': reply.body,
-                    'mediaType': 'text/markdown'
-                },
                 'published': ap_datetime(utcnow()),
                 'distinguished': False,
                 'audience': post.community.public_url(),
@@ -1146,10 +1138,6 @@ def federate_post_update(post):
         'cc': [],
         'content': post.body_html if post.body_html else '',
         'mediaType': 'text/html',
-        'source': {
-            'content': post.body if post.body else '',
-            'mediaType': 'text/markdown'
-        },
         'attachment': [],
         'commentsEnabled': post.comments_enabled,
         'sensitive': post.nsfw,
@@ -1628,10 +1616,6 @@ def post_reply_edit(post_id: int, comment_id: int):
                     'inReplyTo': in_reply_to.profile_id(),
                     'url': post_reply.profile_id(),
                     'mediaType': 'text/html',
-                    'source': {
-                        'content': post_reply.body,
-                        'mediaType': 'text/markdown'
-                    },
                     'published': ap_datetime(post_reply.posted_at),
                     'updated': ap_datetime(post_reply.edited_at),
                     'distinguished': False,

app/templates/post/add_reply.html

@@ -34,7 +34,7 @@
                         });
                     </script>
                 {% else %}
-                    <a href="#" aria-hidden="true" id="post_reply_markdown_editor_enabler" class="markdown_editor_enabler" data-id="body">{{ _('Enable markdown editor') }}</a>
+                    <!-- <a href="#" aria-hidden="true" id="post_reply_markdown_editor_enabler" class="markdown_editor_enabler" data-id="body">{{ _('Enable markdown editor') }}</a> -->
                 {% endif %}
             {% endif %}
         </div>

app/user/routes.py

@@ -108,7 +108,7 @@ def edit_profile(actor):
         if form.password_field.data.strip() != '':
             current_user.set_password(form.password_field.data)
         current_user.about = form.about.data
-        current_user.about_html = allowlist_html(markdown_to_html(form.about.data))
+        current_user.about_html = markdown_to_html(form.about.data)
         current_user.matrix_user_id = form.matrixuserid.data
         current_user.bot = form.bot.data
         profile_file = request.files['profile_file']

app/utils.py

@@ -227,10 +227,24 @@ def allowlist_html(html: str) -> str:
     return re_empty_anchor.sub(r'<a href="\1" rel="nofollow ugc" target="_blank">\1</a>', str(soup))
 
 
+# this is for pyfedi's version of Markdown (differs from lemmy for: newlines for soft breaks, ...)
 def markdown_to_html(markdown_text) -> str:
+    if markdown_text:
+        raw_html = markdown2.markdown(markdown_text, safe_mode=True,
+                    extras={'middle-word-em': False, 'tables': True, 'fenced-code-blocks': True, 'strike': True, 'breaks': {'on_newline': True, 'on_backslash': True}})
+        # support lemmy's spoiler format
+        re_spoiler = re.compile(r':{3}\s*?spoiler\s+?(\S.+?)(?:\n|</p>)(.+?)(?:\n|<p>):{3}', re.S)
+        raw_html = re_spoiler.sub(r'<details><summary>\1</summary><p>\2</p></details>', raw_html)
+        return allowlist_html(raw_html)
+    else:
+        return ''
+
+
+# this is for lemmy's version of Markdown (can be removed in future - when HTML from them filtered through an allow_list is used, instead of MD)
+def lemmy_markdown_to_html(markdown_text) -> str:
     if markdown_text:
         raw_html = markdown2.markdown(markdown_text, safe_mode=True, extras={'middle-word-em': False, 'tables': True, 'fenced-code-blocks': True, 'strike': True})
-        # replace lemmy spoiler tokens with appropriate html tags instead. (until possibly added as extra to markdown2)
+        # replace lemmy spoiler tokens with appropriate html tags instead.
         re_spoiler = re.compile(r':{3}\s*?spoiler\s+?(\S.+?)(?:\n|</p>)(.+?)(?:\n|<p>):{3}', re.S)
         raw_html = re_spoiler.sub(r'<details><summary>\1</summary><p>\2</p></details>', raw_html)
         return allowlist_html(raw_html)