From 4888e2e2e246660b2f02fc62ecccb52992bac801 Mon Sep 17 00:00:00 2001 From: rimu <3310831+rimu@users.noreply.github.com> Date: Sat, 16 Sep 2023 19:09:04 +1200 Subject: [PATCH] federate incoming posts and post replies --- README.md | 12 +- ROADMAP.md | 1 + app/__init__.py | 3 + app/activitypub/routes.py | 315 +++++++++++++++++----------- app/activitypub/util.py | 90 +++++++- app/admin/routes.py | 1 + app/auth/routes.py | 3 +- app/community/routes.py | 4 +- app/main/routes.py | 3 +- app/models.py | 14 +- app/templates/list_communities.html | 10 +- app/utils.py | 95 ++++++++- config.py | 6 + requirements.txt | 2 + 14 files changed, 412 insertions(+), 147 deletions(-) diff --git a/README.md b/README.md index a0f11278..7c6e69cf 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,17 @@ A lemmy/kbin clone written in Python with Flask. - - Clean, simple code. + - Clean, simple code that is easy to understand and contribute to. No fancy design patterns or algorithms. - Easy setup, easy to manage - few dependencies and extra software required. - GPL. + - First class moderation tools. + +## Project goals + +To build a ... + + +## Differences from other federated systems + +... diff --git a/ROADMAP.md b/ROADMAP.md index bd576028..356f80a2 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -11,6 +11,7 @@ The following are the goals for a 1.0 release, good enough for production use. I - vote - sort posts by hotness algo - markdown +- logging and debugging support ### Activitypub-enabled diff --git a/app/__init__.py b/app/__init__.py index c2ded722..0bc53e29 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -12,6 +12,7 @@ from flask_bootstrap import Bootstrap5 from flask_mail import Mail from flask_moment import Moment from flask_babel import Babel, lazy_gettext as _l +from flask_caching import Cache from sqlalchemy_searchable import make_searchable from config import Config @@ -26,6 +27,7 @@ mail = Mail() bootstrap = Bootstrap5() moment = Moment() babel = Babel() +cache = Cache(config={'CACHE_TYPE': os.environ.get('CACHE_TYPE'), 'CACHE_DIR': os.environ.get('CACHE_DIR') or '/dev/shm'}) def create_app(config_class=Config): @@ -40,6 +42,7 @@ def create_app(config_class=Config): moment.init_app(app) make_searchable(db.metadata) babel.init_app(app, locale_selector=get_locale) + cache.init_app(app) from app.main import bp as main_bp app.register_blueprint(main_bp) diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index 3901c636..0bfa0925 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -1,17 +1,20 @@ +import markdown2 import werkzeug.exceptions from sqlalchemy import text from app import db from app.activitypub import bp -from flask import request, Response, render_template, current_app, abort, jsonify, json +from flask import request, Response, current_app, abort, jsonify, json from app.activitypub.signature import HttpSignature from app.community.routes import show_community +from app.constants import POST_TYPE_LINK, POST_TYPE_IMAGE from app.models import User, Community, CommunityJoinRequest, CommunityMember, CommunityBan, ActivityPubLog, Post, \ - PostReply, Instance, PostVote, PostReplyVote + PostReply, Instance, PostVote, PostReplyVote, File from app.activitypub.util import public_key, users_total, active_half_year, active_month, local_posts, local_comments, \ - post_to_activity, find_actor_or_create -from app.utils import gibberish, get_setting + post_to_activity, find_actor_or_create, default_context, instance_blocked, find_reply_parent, find_liked_object +from app.utils import gibberish, get_setting, is_image_url, allowlist_html, html_to_markdown, render_template, \ + domain_from_url INBOX = [] @@ -105,16 +108,7 @@ def user_profile(actor): if user is not None: if 'application/ld+json' in request.headers.get('Accept', '') or request.accept_mimetypes.accept_json: server = current_app.config['SERVER_NAME'] - actor_data = { "@context": [ - "https://www.w3.org/ns/activitystreams", - "https://w3id.org/security/v1", - { - "manuallyApprovesFollowers": "as:manuallyApprovesFollowers", - "schema": "http://schema.org#", - "PropertyValue": "schema:PropertyValue", - "value": "schema:value" - } - ], + actor_data = { "@context": default_context(), "type": "Person", "id": f"https://{server}/u/{actor}", "preferredUsername": actor, @@ -157,10 +151,7 @@ def community_profile(actor): if community is not None: if 'application/ld+json' in request.headers.get('Accept', ''): server = current_app.config['SERVER_NAME'] - actor_data = {"@context": [ - "https://www.w3.org/ns/activitystreams", - "https://w3id.org/security/v1" - ], + actor_data = {"@context": default_context(), "type": "Group", "id": f"https://{server}/c/{actor}", "name": actor.title, @@ -210,6 +201,7 @@ def shared_inbox(): request_json = request.get_json(force=True) except werkzeug.exceptions.BadRequest as e: activity_log.exception_message = 'Unable to parse json body: ' + e.description + activity_log.result = 'failure' db.session.add(activity_log) db.session.commit() return @@ -217,123 +209,205 @@ def shared_inbox(): if 'id' in request_json: activity_log.activity_id = request_json['id'] - actor = find_actor_or_create(request_json['actor']) + actor = find_actor_or_create(request_json['actor']) if 'actor' in request_json else None if actor is not None: if HttpSignature.verify_request(request, actor.public_key, skip_date=True): if 'type' in request_json: activity_log.activity_type = request_json['type'] - if request_json['type'] == 'Announce': - if request_json['object']['type'] == 'Like' or request_json['object']['type'] == 'Dislike': - activity_log.activity_type = request_json['object']['type'] - vote_effect = 1.0 if request_json['object']['type'] == 'Like' else -1.0 - if vote_effect < 0 and get_setting('allow_dislike', True) is False: - activity_log.exception_message = 'Dislike ignored because of allow_dislike setting' - else: - user_ap_id = request_json['object']['actor'] - liked_ap_id = request_json['object']['object'] + if not instance_blocked(request_json['id']): + # Announce is new content and votes + if request_json['type'] == 'Announce': + if request_json['object']['type'] == 'Create': + activity_log.activity_type = request_json['object']['type'] + user_ap_id = request_json['object']['object']['attributedTo'] + community_ap_id = request_json['object']['audience'] + community = find_actor_or_create(community_ap_id) user = find_actor_or_create(user_ap_id) - vote_weight = 1.0 - if user.ap_domain: - instance = Instance.query.filter_by(domain=user.ap_domain).fetch() - if instance: - vote_weight = instance.vote_weight - liked = find_liked_object(liked_ap_id) - # insert into voted table - if isinstance(liked, Post): - existing_vote = PostVote.query.filter_by(user_id=user.id, post_id=liked.id).first() - if existing_vote: - existing_vote.effect = vote_effect * vote_weight + if user and community: + object_type = request_json['object']['object']['type'] + new_content_types = ['Page', 'Article', 'Link', 'Note'] + if object_type in new_content_types: # create a new post + in_reply_to = request_json['object']['object']['inReplyTo'] if 'inReplyTo' in \ + request_json['object']['object'] else None + + if not in_reply_to: + post = Post(user_id=user.id, community_id=community.id, + title=request_json['object']['object']['name'], + comments_enabled=request_json['object']['object']['commentsEnabled'], + sticky=request_json['object']['object']['stickied'] if 'stickied' in request_json['object']['object'] else False, + nsfw=request_json['object']['object']['sensitive'], + nsfl=request_json['object']['object']['nsfl'] if 'nsfl' in request_json['object']['object'] else False, + ap_id=request_json['object']['object']['id'], + ap_create_id=request_json['object']['id'], + ap_announce_id=request_json['id'], + ) + if 'source' in request_json['object']['object'] and \ + request_json['object']['object']['source']['mediaType'] == 'text/markdown': + post.body = request_json['object']['object']['source']['content'] + post.body_html = allowlist_html(markdown2.markdown(post.body, safe_mode=True)) + elif 'content' in request_json['object']['object']: + post.body_html = allowlist_html(request_json['object']['object']['content']) + post.body = html_to_markdown(post.body_html) + if 'attachment' in request_json['object']['object'] and \ + len(request_json['object']['object']['attachment']) > 0 and \ + 'type' in request_json['object']['object']['attachment'][0]: + if request_json['object']['object']['attachment'][0]['type'] == 'Link': + post.url = request_json['object']['object']['attachment'][0]['href'] + if is_image_url(post.url): + post.type = POST_TYPE_IMAGE + else: + post.type = POST_TYPE_LINK + domain = domain_from_url(post.url) + if not domain.banned: + post.domain_id = domain.id + else: + post = None + activity_log.exception_message = domain.name + ' is blocked by admin' + activity_log.result = 'failure' + if 'image' in request_json['object']['object']: + image = File(source_url=request_json['object']['object']['image']['url']) + db.session.add(image) + post.image = image + + if post is not None: + db.session.add(post) + db.session.commit() + else: + post_id, parent_comment_id, root_id = find_reply_parent(in_reply_to) + post_reply = PostReply(user_id=user.id, community_id=community.id, + post_id=post_id, parent_id=parent_comment_id, + root_id=root_id, + nsfw=community.nsfw, + nsfl=community.nsfl, + ap_id=request_json['object']['object']['id'], + ap_create_id=request_json['object']['id'], + ap_announce_id=request_json['id']) + if 'source' in request_json['object']['object'] and \ + request_json['object']['object']['source'][ + 'mediaType'] == 'text/markdown': + post_reply.body = request_json['object']['object']['source']['content'] + post_reply.body_html = allowlist_html(markdown2.markdown(post_reply.body, safe_mode=True)) + elif 'content' in request_json['object']['object']: + post_reply.body_html = allowlist_html( + request_json['object']['object']['content']) + post_reply.body = html_to_markdown(post_reply.body_html) + + if post_reply is not None: + db.session.add(post_reply) + db.session.commit() else: - vote = PostVote(user_id=user.id, author_id=liked.user_id, post_id=liked.id, - effect=vote_effect * vote_weight) - db.session.add(vote) - db.session.commit() - activity_log.result = 'success' - elif isinstance(liked, PostReply): - existing_vote = PostVote.query.filter_by(user_id=user.id, post_id=liked.id).first() - if existing_vote: - existing_vote.effect = vote_effect * vote_weight + activity_log.exception_message = 'Unacceptable type: ' + object_type + + elif request_json['object']['type'] == 'Like' or request_json['object']['type'] == 'Dislike': + activity_log.activity_type = request_json['object']['type'] + vote_effect = 1.0 if request_json['object']['type'] == 'Like' else -1.0 + if vote_effect < 0 and get_setting('allow_dislike', True) is False: + activity_log.exception_message = 'Dislike ignored because of allow_dislike setting' + else: + user_ap_id = request_json['object']['actor'] + liked_ap_id = request_json['object']['object'] + user = find_actor_or_create(user_ap_id) + vote_weight = 1.0 + if user.ap_domain: + instance = Instance.query.filter_by(domain=user.ap_domain).fetch() + if instance: + vote_weight = instance.vote_weight + liked = find_liked_object(liked_ap_id) + # insert into voted table + if liked is not None and isinstance(liked, Post): + existing_vote = PostVote.query.filter_by(user_id=user.id, post_id=liked.id).first() + if existing_vote: + existing_vote.effect = vote_effect * vote_weight + else: + vote = PostVote(user_id=user.id, author_id=liked.user_id, post_id=liked.id, + effect=vote_effect * vote_weight) + db.session.add(vote) + db.session.commit() + activity_log.result = 'success' + elif liked is not None and isinstance(liked, PostReply): + existing_vote = PostVote.query.filter_by(user_id=user.id, post_id=liked.id).first() + if existing_vote: + existing_vote.effect = vote_effect * vote_weight + else: + vote = PostReplyVote(user_id=user.id, author_id=liked.user_id, post_reply_id=liked.id, + effect=vote_effect * vote_weight) + db.session.add(vote) + db.session.commit() + activity_log.result = 'success' else: - vote = PostReplyVote(user_id=user.id, author_id=liked.user_id, post_reply_id=liked.id, - effect=vote_effect * vote_weight) - db.session.add(vote) - db.session.commit() + activity_log.exception_message = 'Could not detect type of like' + if activity_log.result == 'success': + ... # todo: recalculate 'hotness' of liked post/reply + + # Follow: remote user wants to follow one of our communities + elif request_json['type'] == 'Follow': # Follow is when someone wants to join a community + user_ap_id = request_json['actor'] + community_ap_id = request_json['object'] + follow_id = request_json['id'] + user = find_actor_or_create(user_ap_id) + community = find_actor_or_create(community_ap_id) + if user is not None and community is not None: + # check if user is banned from this community + banned = CommunityBan.query.filter_by(user_id=user.id, + community_id=community.id).first() + if banned is None: + if not user.subscribed(community): + member = CommunityMember(user_id=user.id, community_id=community.id) + db.session.add(member) + db.session.commit() + # send accept message to acknowledge the follow + accept = { + "@context": default_context(), + "actor": community.ap_profile_id, + "to": [ + user.ap_profile_id + ], + "object": { + "actor": user.ap_profile_id, + "to": None, + "object": community.ap_profile_id, + "type": "Follow", + "id": follow_id + }, + "type": "Accept", + "id": f"https://{current_app.config['SERVER_NAME']}/activities/accept/" + gibberish(32) + } + try: + HttpSignature.signed_request(user.ap_inbox_url, accept, community.private_key, + f"https://{current_app.config['SERVER_NAME']}/c/{community.name}#main-key") + except Exception as e: + accept_log = ActivityPubLog(direction='out', activity_json=json.dumps(accept), + result='failure', activity_id=accept['id'], + exception_message = 'could not send Accept' + str(e)) + db.session.add(accept_log) + db.session.commit() + return activity_log.result = 'success' else: - activity_log.result='failure' - activity_log.exception_message = 'Could not detect type of like' - if activity_log.result == 'success': - ... # todo: recalculate 'hotness' of liked post/reply - - # remote user wants to follow one of our communities - elif request_json['type'] == 'Follow': - user_ap_id = request_json['actor'] - community_ap_id = request_json['object'] - follow_id = request_json['id'] - user = find_actor_or_create(user_ap_id) - community = find_actor_or_create(community_ap_id) - if user is not None and community is not None: - # check if user is banned from this community - banned = CommunityBan.query.filter_by(user_id=user.id, - community_id=community.id).first() - if banned is None: - if not user.subscribed(community): + activity_log.exception_message = 'user is banned from this community' + # Accept: remote server is accepting our previous follow request + elif request_json['type'] == 'Accept': + if request_json['object']['type'] == 'Follow': + community_ap_id = request_json['actor'] + user_ap_id = request_json['object']['actor'] + user = find_actor_or_create(user_ap_id) + community = find_actor_or_create(community_ap_id) + join_request = CommunityJoinRequest.query.filter_by(user_id=user.id, + community_id=community.id).first() + if join_request: member = CommunityMember(user_id=user.id, community_id=community.id) db.session.add(member) db.session.commit() - # send accept message to acknowledge the follow - accept = { - "@context": [ - "https://www.w3.org/ns/activitystreams", - "https://w3id.org/security/v1", - ], - "actor": community.ap_profile_id, - "to": [ - user.ap_profile_id - ], - "object": { - "actor": user.ap_profile_id, - "to": None, - "object": community.ap_profile_id, - "type": "Follow", - "id": follow_id - }, - "type": "Accept", - "id": f"https://{current_app.config['SERVER_NAME']}/activities/accept/" + gibberish(32) - } - try: - HttpSignature.signed_request(user.ap_inbox_url, accept, community.private_key, - f"https://{current_app.config['SERVER_NAME']}/c/{community.name}#main-key") - except Exception as e: - accept_log = ActivityPubLog(direction='out', activity_json=json.dumps(accept), - result='failure', activity_id=accept['id'], - exception_message = 'could not send Accept' + str(e)) - db.session.add(accept_log) - db.session.commit() - return - activity_log.result = 'success' - else: - activity_log.exception_message = 'user is banned from this community' - # remote server is accepting our previous follow request - elif request_json['type'] == 'Accept': - if request_json['object']['type'] == 'Follow': - community_ap_id = request_json['actor'] - user_ap_id = request_json['object']['actor'] - user = find_actor_or_create(user_ap_id) - community = find_actor_or_create(community_ap_id) - join_request = CommunityJoinRequest.query.filter_by(user_id=user.id, - community_id=community.id).first() - if join_request: - member = CommunityMember(user_id=user.id, community_id=community.id) - db.session.add(member) - db.session.commit() - activity_log.result = 'success' - + activity_log.result = 'success' + else: + activity_log.exception_message = 'Instance banned' else: activity_log.exception_message = 'Could not verify signature' else: activity_log.exception_message = 'Actor could not be found: ' + request_json['actor'] + if activity_log.exception_message is not None: + activity_log.result = 'failure' db.session.add(activity_log) db.session.commit() @@ -346,10 +420,7 @@ def community_outbox(actor): posts = community.posts.limit(50).all() community_data = { - "@context": [ - "https://www.w3.org/ns/activitystreams", - "https://w3id.org/security/v1", - ], + "@context": default_context(), "type": "OrderedCollection", "id": f"https://{current_app.config['SERVER_NAME']}/c/{actor}/outbox", "totalItems": len(posts), diff --git a/app/activitypub/util.py b/app/activitypub/util.py index 61b3e238..937a38d3 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -1,22 +1,21 @@ import json import os from datetime import datetime -from typing import Union +from typing import Union, Tuple import markdown2 from flask import current_app from sqlalchemy import text -from app import db -from app.models import User, Post, Community, BannedInstances, File +from app import db, cache +from app.models import User, Post, Community, BannedInstances, File, PostReply import time import base64 import requests from cryptography.hazmat.primitives import serialization, hashes from cryptography.hazmat.primitives.asymmetric import padding from app.constants import * -import functools from urllib.parse import urlparse -from app.utils import get_request +from app.utils import get_request, allowlist_html def public_key(): @@ -177,8 +176,11 @@ def banned_user_agents(): return [] # todo: finish this function -@functools.lru_cache(maxsize=100) -def instance_blocked(host): +@cache.cached(150) +def instance_blocked(host: str) -> bool: + host = host.lower() + if 'https://' in host or 'http://' in host: + host = urlparse(host).hostname instance = BannedInstances.query.filter_by(domain=host.strip()).first() return instance is not None @@ -198,7 +200,8 @@ def find_actor_or_create(actor: str) -> Union[User, Community, None]: server, address = extract_domain_and_actor(actor) if instance_blocked(server): return None - user = User.query.filter_by(ap_profile_id=actor).first() # finds users formatted like https://kbin.social/u/tables + user = User.query.filter_by( + ap_profile_id=actor).first() # finds users formatted like https://kbin.social/u/tables if user is None: user = Community.query.filter_by(ap_profile_id=actor).first() if user is None: @@ -301,6 +304,75 @@ def parse_summary(user_json) -> str: html_content = markdown2.markdown(markdown_text) return html_content elif 'summary' in user_json: - return user_json['summary'] + return allowlist_html(user_json['summary']) else: return '' + + +def default_context(): + context = [ + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + ] + if current_app.config['FULL_AP_CONTEXT']: + context.append({ + "lemmy": "https://join-lemmy.org/ns#", + "litepub": "http://litepub.social/ns#", + "pt": "https://joinpeertube.org/ns#", + "sc": "http://schema.org/", + "ChatMessage": "litepub:ChatMessage", + "commentsEnabled": "pt:commentsEnabled", + "sensitive": "as:sensitive", + "matrixUserId": "lemmy:matrixUserId", + "postingRestrictedToMods": "lemmy:postingRestrictedToMods", + "removeData": "lemmy:removeData", + "stickied": "lemmy:stickied", + "moderators": { + "@type": "@id", + "@id": "lemmy:moderators" + }, + "expires": "as:endTime", + "distinguished": "lemmy:distinguished", + "language": "sc:inLanguage", + "identifier": "sc:identifier" + }) + return context + + +def find_reply_parent(in_reply_to: str) -> Tuple[int, int, int]: + if 'comment' in in_reply_to: + parent_comment = PostReply.get_by_ap_id(in_reply_to) + parent_comment_id = parent_comment.id + post_id = parent_comment.post_id + root_id = parent_comment.root_id + elif 'post' in in_reply_to: + parent_comment_id = None + post = Post.get_by_ap_id(in_reply_to) + post_id = post.id + root_id = None + else: + parent_comment_id = None + root_id = None + post_id = None + post = Post.get_by_ap_id(in_reply_to) + if post: + post_id = post.id + else: + parent_comment = PostReply.get_by_ap_id(in_reply_to) + if parent_comment: + parent_comment_id = parent_comment.id + post_id = parent_comment.post_id + root_id = parent_comment.root_id + + return post_id, parent_comment_id, root_id + + +def find_liked_object(ap_id) -> Union[Post, PostReply, None]: + post = Post.get_by_ap_id(ap_id) + if post: + return post + else: + post_reply = PostReply.get_by_ap_id(ap_id) + if post_reply: + return post_reply + return None diff --git a/app/admin/routes.py b/app/admin/routes.py index e69de29b..52c1dba0 100644 --- a/app/admin/routes.py +++ b/app/admin/routes.py @@ -0,0 +1 @@ +from app.utils import render_template \ No newline at end of file diff --git a/app/auth/routes.py b/app/auth/routes.py index e5537e01..7a88cc4d 100644 --- a/app/auth/routes.py +++ b/app/auth/routes.py @@ -1,5 +1,5 @@ from datetime import date, datetime, timedelta -from flask import render_template, redirect, url_for, flash, request, make_response, session, Markup, current_app +from flask import redirect, url_for, flash, request, make_response, session, Markup, current_app from werkzeug.urls import url_parse from flask_login import login_user, logout_user, current_user from flask_babel import _ @@ -10,6 +10,7 @@ from app.auth.util import random_token from app.models import User from app.auth.email import send_password_reset_email, send_welcome_email, send_verification_email from app.activitypub.signature import RsaKeys +from app.utils import render_template @bp.route('/login', methods=['GET', 'POST']) diff --git a/app/community/routes.py b/app/community/routes.py index 04118966..19a56d2a 100644 --- a/app/community/routes.py +++ b/app/community/routes.py @@ -1,5 +1,5 @@ from datetime import date, datetime, timedelta -from flask import render_template, redirect, url_for, flash, request, make_response, session, Markup, current_app, abort +from flask import redirect, url_for, flash, request, make_response, session, Markup, current_app, abort from flask_login import login_user, logout_user, current_user from flask_babel import _ from app import db @@ -9,7 +9,7 @@ from app.community.util import search_for_community, community_url_exists from app.constants import SUBSCRIPTION_MEMBER, SUBSCRIPTION_OWNER from app.models import User, Community, CommunityMember, CommunityJoinRequest, CommunityBan from app.community import bp -from app.utils import get_setting +from app.utils import get_setting, render_template from sqlalchemy import or_ diff --git a/app/main/routes.py b/app/main/routes.py index efae5c33..c5588f49 100644 --- a/app/main/routes.py +++ b/app/main/routes.py @@ -2,12 +2,13 @@ from datetime import datetime from app import db from app.main import bp -from flask import g, jsonify, render_template, flash, request +from flask import g, jsonify, flash, request from flask_moment import moment from flask_login import current_user from flask_babel import _, get_locale from sqlalchemy import select from sqlalchemy_searchable import search +from app.utils import render_template from app.models import Community, CommunityMember diff --git a/app/models.py b/app/models.py index 103436d4..49ba82bf 100644 --- a/app/models.py +++ b/app/models.py @@ -231,6 +231,7 @@ class Post(db.Model): body = db.Column(db.Text) body_html = db.Column(db.Text) type = db.Column(db.Integer) + comments_enabled = db.Column(db.Boolean, default=True) has_embed = db.Column(db.Boolean, default=False) reply_count = db.Column(db.Integer, default=0) score = db.Column(db.Integer, default=0, index=True) @@ -256,6 +257,10 @@ class Post(db.Model): image = db.relationship(File, foreign_keys=[image_id], cascade="all, delete") + @classmethod + def get_by_ap_id(cls, ap_id): + return cls.query.filter_by(ap_id=ap_id).first() + class PostReply(db.Model): id = db.Column(db.Integer, primary_key=True) @@ -267,6 +272,7 @@ class PostReply(db.Model): root_id = db.Column(db.Integer) body = db.Column(db.Text) body_html = db.Column(db.Text) + body_html_safe = db.Column(db.Boolean, default=False) score = db.Column(db.Integer, default=0, index=True) nsfw = db.Column(db.Boolean, default=False) nsfl = db.Column(db.Boolean, default=False) @@ -280,15 +286,21 @@ class PostReply(db.Model): edited_at = db.Column(db.DateTime) ap_id = db.Column(db.String(255), index=True) + ap_create_id = db.Column(db.String(100)) + ap_announce_id = db.Column(db.String(100)) search_vector = db.Column(TSVectorType('body')) + @classmethod + def get_by_ap_id(cls, ap_id): + return cls.query.filter_by(ap_id=ap_id).first() + class Domain(db.Model): id = db.Column(db.Integer, primary_key=True) name = db.Column(db.String(255), index=True) post_count = db.Column(db.Integer, default=0) - banned = db.Column(db.Boolean, default=False, index=True) + banned = db.Column(db.Boolean, default=False, index=True) # Domains can be banned site-wide (by admin) or DomainBlock'ed by users class DomainBlock(db.Model): diff --git a/app/templates/list_communities.html b/app/templates/list_communities.html index 4ed4d961..483ae694 100644 --- a/app/templates/list_communities.html +++ b/app/templates/list_communities.html @@ -49,10 +49,12 @@ {{ community.post_count }} {{ community.post_reply_count }} {{ moment(community.last_active).fromNow(refresh=True) }} - {% if current_user.subscribed(community) %} - Unsubscribe - {% else %} - Subscribe + {% if current_user.is_authenticated %} + {% if current_user.subscribed(community) %} + Unsubscribe + {% else %} + Subscribe + {% endif %} {% endif %} diff --git a/app/utils.py b/app/utils.py index 60ebd274..19ef9d96 100644 --- a/app/utils.py +++ b/app/utils.py @@ -1,13 +1,26 @@ import functools import random +from urllib.parse import urlparse + +import flask +from bs4 import BeautifulSoup +import html as html_module import requests import os from flask import current_app, json -from app import db -from app.models import Settings +from app import db, cache +from app.models import Settings, Domain, Instance, BannedInstances + + +# Flask's render_template function, with support for themes added +def render_template(template_name: str, **context) -> str: + theme = get_setting('theme', '') + if theme != '': + return flask.render_template(f'themes/{theme}/{template_name}', **context) + else: + return flask.render_template(template_name, **context) -# ---------------------------------------------------------------------- # Jinja: when a file was modified. Useful for cache-busting def getmtime(filename): return os.path.getmtime('static/' + filename) @@ -28,8 +41,8 @@ def get_request(uri, params=None, headers=None) -> requests.Response: return response -# saves an arbitrary object into a persistent key-value store. Possibly redis would be faster than using the DB -@functools.lru_cache(maxsize=100) +# saves an arbitrary object into a persistent key-value store. cached. +@cache.cached(timeout=50) def get_setting(name: str, default=None): setting = Settings.query.filter_by(name=name).first() if setting is None: @@ -46,7 +59,7 @@ def set_setting(name: str, value): else: setting.value = json.dumps(value) db.session.commit() - get_setting.cache_clear() + cache.delete_memoized(get_setting) # Return the contents of a file as a string. Inspired by PHP's function of the same name. @@ -61,3 +74,73 @@ random_chars = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' def gibberish(length: int = 10) -> str: return "".join([random.choice(random_chars) for x in range(length)]) + + +def is_image_url(url): + parsed_url = urlparse(url) + path = parsed_url.path.lower() + common_image_extensions = ['.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.webp'] + return any(path.endswith(extension) for extension in common_image_extensions) + + +# sanitise HTML using an allow list +def allowlist_html(html: str) -> str: + allowed_tags = ['p', 'strong', 'a', 'ul', 'ol', 'li', 'em', 'blockquote', 'cite', 'br', 'h3', 'h4', 'h5'] + # Parse the HTML using BeautifulSoup + soup = BeautifulSoup(html, 'html.parser') + + # Find all tags in the parsed HTML + for tag in soup.find_all(): + # If the tag is not in the allowed_tags list, remove it and its contents + if tag.name not in allowed_tags: + tag.extract() + else: + # Filter and sanitize attributes + for attr in list(tag.attrs): + if attr not in ['href', 'src']: # Add allowed attributes here + del tag[attr] + + # Encode the HTML to prevent script execution + return html_module.escape(str(soup)) + + +# convert basic HTML to Markdown +def html_to_markdown(html: str) -> str: + soup = BeautifulSoup(html, 'html.parser') + return html_to_markdown_worker(soup) + + +def html_to_markdown_worker(element, indent_level=0): + formatted_text = '' + for item in element.contents: + if isinstance(item, str): + formatted_text += item + elif item.name == 'p': + formatted_text += '\n\n' + elif item.name == 'br': + formatted_text += ' \n' # Double space at the end for line break + elif item.name == 'strong': + formatted_text += '**' + html_to_markdown_worker(item) + '**' + elif item.name == 'ul': + formatted_text += '\n' + formatted_text += html_to_markdown_worker(item, indent_level + 1) + formatted_text += '\n' + elif item.name == 'ol': + formatted_text += '\n' + formatted_text += html_to_markdown_worker(item, indent_level + 1) + formatted_text += '\n' + elif item.name == 'li': + bullet = '-' if item.find_parent(['ul', 'ol']) and item.find_previous_sibling() is None else '' + formatted_text += ' ' * indent_level + bullet + ' ' + html_to_markdown_worker(item).strip() + '\n' + elif item.name == 'blockquote': + formatted_text += ' ' * indent_level + '> ' + html_to_markdown_worker(item).strip() + '\n' + elif item.name == 'code': + formatted_text += '`' + html_to_markdown_worker(item) + '`' + return formatted_text + + +def domain_from_url(url: str) -> Domain: + parsed_url = urlparse(url) + domain = Domain.query.filter_by(name=parsed_url.hostname.lower()).first() + return domain + diff --git a/config.py b/config.py index b4704b35..d1a30e47 100644 --- a/config.py +++ b/config.py @@ -21,3 +21,9 @@ class Config(object): RECAPTCHA3_PRIVATE_KEY = os.environ.get("RECAPTCHA3_PRIVATE_KEY") MODE = os.environ.get('MODE') or 'development' LANGUAGES = ['en'] + FULL_AP_CONTEXT = os.environ.get('FULL_AP_CONTEXT') or True + CACHE_TYPE = os.environ.get('CACHE_TYPE') or 'FileSystemCache' + CACHE_DIR = os.environ.get('CACHE_DIR') or '/dev/shm/pyfedi' + CACHE_DEFAULT_TIMEOUT = 300 + CACHE_THRESHOLD = 1000 + CACHE_KEY_PREFIX = 'pyfedi' diff --git a/requirements.txt b/requirements.txt index a2b926e6..ced1ade6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,3 +20,5 @@ arrow==1.2.3 pyld==2.0.3 boto3==1.28.35 markdown2==2.4.10 +beautifulsoup4==4.12.2 +flask-caching==2.0.2