mirror of
https://codeberg.org/rimu/pyfedi
synced 2025-01-23 11:26:56 -08:00
federate incoming posts and post replies
This commit is contained in:
rimu
parent
08a771daf0
commit
4888e2e2e2
14 changed files with 412 additions and 147 deletions
12
README.md
12
README.md
|
|
@ -2,7 +2,17 @@
|
|||
|
||||
A lemmy/kbin clone written in Python with Flask.
|
||||
|
||||
- Clean, simple code.
|
||||
- Clean, simple code that is easy to understand and contribute to. No fancy design patterns or algorithms.
|
||||
- Easy setup, easy to manage - few dependencies and extra software required.
|
||||
- GPL.
|
||||
- First class moderation tools.
|
||||
|
||||
## Project goals
|
||||
|
||||
To build a ...
|
||||
|
||||
|
||||
## Differences from other federated systems
|
||||
|
||||
...
|
||||
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ The following are the goals for a 1.0 release, good enough for production use. I
|
|||
- vote
|
||||
- sort posts by hotness algo
|
||||
- markdown
|
||||
- logging and debugging support
|
||||
|
||||
|
||||
### Activitypub-enabled
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from flask_bootstrap import Bootstrap5
|
|||
from flask_mail import Mail
|
||||
from flask_moment import Moment
|
||||
from flask_babel import Babel, lazy_gettext as _l
|
||||
from flask_caching import Cache
|
||||
from sqlalchemy_searchable import make_searchable
|
||||
|
||||
from config import Config
|
||||
|
|
@ -26,6 +27,7 @@ mail = Mail()
|
|||
bootstrap = Bootstrap5()
|
||||
moment = Moment()
|
||||
babel = Babel()
|
||||
cache = Cache(config={'CACHE_TYPE': os.environ.get('CACHE_TYPE'), 'CACHE_DIR': os.environ.get('CACHE_DIR') or '/dev/shm'})
|
||||
|
||||
|
||||
def create_app(config_class=Config):
|
||||
|
|
@ -40,6 +42,7 @@ def create_app(config_class=Config):
|
|||
moment.init_app(app)
|
||||
make_searchable(db.metadata)
|
||||
babel.init_app(app, locale_selector=get_locale)
|
||||
cache.init_app(app)
|
||||
|
||||
from app.main import bp as main_bp
|
||||
app.register_blueprint(main_bp)
|
||||
|
|
|
|||
|
|
@ -1,17 +1,20 @@
|
|||
import markdown2
|
||||
import werkzeug.exceptions
|
||||
from sqlalchemy import text
|
||||
|
||||
from app import db
|
||||
from app.activitypub import bp
|
||||
from flask import request, Response, render_template, current_app, abort, jsonify, json
|
||||
from flask import request, Response, current_app, abort, jsonify, json
|
||||
|
||||
from app.activitypub.signature import HttpSignature
|
||||
from app.community.routes import show_community
|
||||
from app.constants import POST_TYPE_LINK, POST_TYPE_IMAGE
|
||||
from app.models import User, Community, CommunityJoinRequest, CommunityMember, CommunityBan, ActivityPubLog, Post, \
|
||||
PostReply, Instance, PostVote, PostReplyVote
|
||||
PostReply, Instance, PostVote, PostReplyVote, File
|
||||
from app.activitypub.util import public_key, users_total, active_half_year, active_month, local_posts, local_comments, \
|
||||
post_to_activity, find_actor_or_create
|
||||
from app.utils import gibberish, get_setting
|
||||
post_to_activity, find_actor_or_create, default_context, instance_blocked, find_reply_parent, find_liked_object
|
||||
from app.utils import gibberish, get_setting, is_image_url, allowlist_html, html_to_markdown, render_template, \
|
||||
domain_from_url
|
||||
|
||||
INBOX = []
|
||||
|
||||
|
|
@ -105,16 +108,7 @@ def user_profile(actor):
|
|||
if user is not None:
|
||||
if 'application/ld+json' in request.headers.get('Accept', '') or request.accept_mimetypes.accept_json:
|
||||
server = current_app.config['SERVER_NAME']
|
||||
actor_data = { "@context": [
|
||||
"https://www.w3.org/ns/activitystreams",
|
||||
"https://w3id.org/security/v1",
|
||||
{
|
||||
"manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
|
||||
"schema": "http://schema.org#",
|
||||
"PropertyValue": "schema:PropertyValue",
|
||||
"value": "schema:value"
|
||||
}
|
||||
],
|
||||
actor_data = { "@context": default_context(),
|
||||
"type": "Person",
|
||||
"id": f"https://{server}/u/{actor}",
|
||||
"preferredUsername": actor,
|
||||
|
|
@ -157,10 +151,7 @@ def community_profile(actor):
|
|||
if community is not None:
|
||||
if 'application/ld+json' in request.headers.get('Accept', ''):
|
||||
server = current_app.config['SERVER_NAME']
|
||||
actor_data = {"@context": [
|
||||
"https://www.w3.org/ns/activitystreams",
|
||||
"https://w3id.org/security/v1"
|
||||
],
|
||||
actor_data = {"@context": default_context(),
|
||||
"type": "Group",
|
||||
"id": f"https://{server}/c/{actor}",
|
||||
"name": actor.title,
|
||||
|
|
@ -210,6 +201,7 @@ def shared_inbox():
|
|||
request_json = request.get_json(force=True)
|
||||
except werkzeug.exceptions.BadRequest as e:
|
||||
activity_log.exception_message = 'Unable to parse json body: ' + e.description
|
||||
activity_log.result = 'failure'
|
||||
db.session.add(activity_log)
|
||||
db.session.commit()
|
||||
return
|
||||
|
|
@ -217,123 +209,205 @@ def shared_inbox():
|
|||
if 'id' in request_json:
|
||||
activity_log.activity_id = request_json['id']
|
||||
|
||||
actor = find_actor_or_create(request_json['actor'])
|
||||
actor = find_actor_or_create(request_json['actor']) if 'actor' in request_json else None
|
||||
if actor is not None:
|
||||
if HttpSignature.verify_request(request, actor.public_key, skip_date=True):
|
||||
if 'type' in request_json:
|
||||
activity_log.activity_type = request_json['type']
|
||||
if request_json['type'] == 'Announce':
|
||||
if request_json['object']['type'] == 'Like' or request_json['object']['type'] == 'Dislike':
|
||||
activity_log.activity_type = request_json['object']['type']
|
||||
vote_effect = 1.0 if request_json['object']['type'] == 'Like' else -1.0
|
||||
if vote_effect < 0 and get_setting('allow_dislike', True) is False:
|
||||
activity_log.exception_message = 'Dislike ignored because of allow_dislike setting'
|
||||
else:
|
||||
user_ap_id = request_json['object']['actor']
|
||||
liked_ap_id = request_json['object']['object']
|
||||
if not instance_blocked(request_json['id']):
|
||||
# Announce is new content and votes
|
||||
if request_json['type'] == 'Announce':
|
||||
if request_json['object']['type'] == 'Create':
|
||||
activity_log.activity_type = request_json['object']['type']
|
||||
user_ap_id = request_json['object']['object']['attributedTo']
|
||||
community_ap_id = request_json['object']['audience']
|
||||
community = find_actor_or_create(community_ap_id)
|
||||
user = find_actor_or_create(user_ap_id)
|
||||
vote_weight = 1.0
|
||||
if user.ap_domain:
|
||||
instance = Instance.query.filter_by(domain=user.ap_domain).fetch()
|
||||
if instance:
|
||||
vote_weight = instance.vote_weight
|
||||
liked = find_liked_object(liked_ap_id)
|
||||
# insert into voted table
|
||||
if isinstance(liked, Post):
|
||||
existing_vote = PostVote.query.filter_by(user_id=user.id, post_id=liked.id).first()
|
||||
if existing_vote:
|
||||
existing_vote.effect = vote_effect * vote_weight
|
||||
if user and community:
|
||||
object_type = request_json['object']['object']['type']
|
||||
new_content_types = ['Page', 'Article', 'Link', 'Note']
|
||||
if object_type in new_content_types: # create a new post
|
||||
in_reply_to = request_json['object']['object']['inReplyTo'] if 'inReplyTo' in \
|
||||
request_json['object']['object'] else None
|
||||
|
||||
if not in_reply_to:
|
||||
post = Post(user_id=user.id, community_id=community.id,
|
||||
title=request_json['object']['object']['name'],
|
||||
comments_enabled=request_json['object']['object']['commentsEnabled'],
|
||||
sticky=request_json['object']['object']['stickied'] if 'stickied' in request_json['object']['object'] else False,
|
||||
nsfw=request_json['object']['object']['sensitive'],
|
||||
nsfl=request_json['object']['object']['nsfl'] if 'nsfl' in request_json['object']['object'] else False,
|
||||
ap_id=request_json['object']['object']['id'],
|
||||
ap_create_id=request_json['object']['id'],
|
||||
ap_announce_id=request_json['id'],
|
||||
)
|
||||
if 'source' in request_json['object']['object'] and \
|
||||
request_json['object']['object']['source']['mediaType'] == 'text/markdown':
|
||||
post.body = request_json['object']['object']['source']['content']
|
||||
post.body_html = allowlist_html(markdown2.markdown(post.body, safe_mode=True))
|
||||
elif 'content' in request_json['object']['object']:
|
||||
post.body_html = allowlist_html(request_json['object']['object']['content'])
|
||||
post.body = html_to_markdown(post.body_html)
|
||||
if 'attachment' in request_json['object']['object'] and \
|
||||
len(request_json['object']['object']['attachment']) > 0 and \
|
||||
'type' in request_json['object']['object']['attachment'][0]:
|
||||
if request_json['object']['object']['attachment'][0]['type'] == 'Link':
|
||||
post.url = request_json['object']['object']['attachment'][0]['href']
|
||||
if is_image_url(post.url):
|
||||
post.type = POST_TYPE_IMAGE
|
||||
else:
|
||||
post.type = POST_TYPE_LINK
|
||||
domain = domain_from_url(post.url)
|
||||
if not domain.banned:
|
||||
post.domain_id = domain.id
|
||||
else:
|
||||
post = None
|
||||
activity_log.exception_message = domain.name + ' is blocked by admin'
|
||||
activity_log.result = 'failure'
|
||||
if 'image' in request_json['object']['object']:
|
||||
image = File(source_url=request_json['object']['object']['image']['url'])
|
||||
db.session.add(image)
|
||||
post.image = image
|
||||
|
||||
if post is not None:
|
||||
db.session.add(post)
|
||||
db.session.commit()
|
||||
else:
|
||||
post_id, parent_comment_id, root_id = find_reply_parent(in_reply_to)
|
||||
post_reply = PostReply(user_id=user.id, community_id=community.id,
|
||||
post_id=post_id, parent_id=parent_comment_id,
|
||||
root_id=root_id,
|
||||
nsfw=community.nsfw,
|
||||
nsfl=community.nsfl,
|
||||
ap_id=request_json['object']['object']['id'],
|
||||
ap_create_id=request_json['object']['id'],
|
||||
ap_announce_id=request_json['id'])
|
||||
if 'source' in request_json['object']['object'] and \
|
||||
request_json['object']['object']['source'][
|
||||
'mediaType'] == 'text/markdown':
|
||||
post_reply.body = request_json['object']['object']['source']['content']
|
||||
post_reply.body_html = allowlist_html(markdown2.markdown(post_reply.body, safe_mode=True))
|
||||
elif 'content' in request_json['object']['object']:
|
||||
post_reply.body_html = allowlist_html(
|
||||
request_json['object']['object']['content'])
|
||||
post_reply.body = html_to_markdown(post_reply.body_html)
|
||||
|
||||
if post_reply is not None:
|
||||
db.session.add(post_reply)
|
||||
db.session.commit()
|
||||
else:
|
||||
vote = PostVote(user_id=user.id, author_id=liked.user_id, post_id=liked.id,
|
||||
effect=vote_effect * vote_weight)
|
||||
db.session.add(vote)
|
||||
db.session.commit()
|
||||
activity_log.result = 'success'
|
||||
elif isinstance(liked, PostReply):
|
||||
existing_vote = PostVote.query.filter_by(user_id=user.id, post_id=liked.id).first()
|
||||
if existing_vote:
|
||||
existing_vote.effect = vote_effect * vote_weight
|
||||
activity_log.exception_message = 'Unacceptable type: ' + object_type
|
||||
|
||||
elif request_json['object']['type'] == 'Like' or request_json['object']['type'] == 'Dislike':
|
||||
activity_log.activity_type = request_json['object']['type']
|
||||
vote_effect = 1.0 if request_json['object']['type'] == 'Like' else -1.0
|
||||
if vote_effect < 0 and get_setting('allow_dislike', True) is False:
|
||||
activity_log.exception_message = 'Dislike ignored because of allow_dislike setting'
|
||||
else:
|
||||
user_ap_id = request_json['object']['actor']
|
||||
liked_ap_id = request_json['object']['object']
|
||||
user = find_actor_or_create(user_ap_id)
|
||||
vote_weight = 1.0
|
||||
if user.ap_domain:
|
||||
instance = Instance.query.filter_by(domain=user.ap_domain).fetch()
|
||||
if instance:
|
||||
vote_weight = instance.vote_weight
|
||||
liked = find_liked_object(liked_ap_id)
|
||||
# insert into voted table
|
||||
if liked is not None and isinstance(liked, Post):
|
||||
existing_vote = PostVote.query.filter_by(user_id=user.id, post_id=liked.id).first()
|
||||
if existing_vote:
|
||||
existing_vote.effect = vote_effect * vote_weight
|
||||
else:
|
||||
vote = PostVote(user_id=user.id, author_id=liked.user_id, post_id=liked.id,
|
||||
effect=vote_effect * vote_weight)
|
||||
db.session.add(vote)
|
||||
db.session.commit()
|
||||
activity_log.result = 'success'
|
||||
elif liked is not None and isinstance(liked, PostReply):
|
||||
existing_vote = PostVote.query.filter_by(user_id=user.id, post_id=liked.id).first()
|
||||
if existing_vote:
|
||||
existing_vote.effect = vote_effect * vote_weight
|
||||
else:
|
||||
vote = PostReplyVote(user_id=user.id, author_id=liked.user_id, post_reply_id=liked.id,
|
||||
effect=vote_effect * vote_weight)
|
||||
db.session.add(vote)
|
||||
db.session.commit()
|
||||
activity_log.result = 'success'
|
||||
else:
|
||||
vote = PostReplyVote(user_id=user.id, author_id=liked.user_id, post_reply_id=liked.id,
|
||||
effect=vote_effect * vote_weight)
|
||||
db.session.add(vote)
|
||||
db.session.commit()
|
||||
activity_log.exception_message = 'Could not detect type of like'
|
||||
if activity_log.result == 'success':
|
||||
... # todo: recalculate 'hotness' of liked post/reply
|
||||
|
||||
# Follow: remote user wants to follow one of our communities
|
||||
elif request_json['type'] == 'Follow': # Follow is when someone wants to join a community
|
||||
user_ap_id = request_json['actor']
|
||||
community_ap_id = request_json['object']
|
||||
follow_id = request_json['id']
|
||||
user = find_actor_or_create(user_ap_id)
|
||||
community = find_actor_or_create(community_ap_id)
|
||||
if user is not None and community is not None:
|
||||
# check if user is banned from this community
|
||||
banned = CommunityBan.query.filter_by(user_id=user.id,
|
||||
community_id=community.id).first()
|
||||
if banned is None:
|
||||
if not user.subscribed(community):
|
||||
member = CommunityMember(user_id=user.id, community_id=community.id)
|
||||
db.session.add(member)
|
||||
db.session.commit()
|
||||
# send accept message to acknowledge the follow
|
||||
accept = {
|
||||
"@context": default_context(),
|
||||
"actor": community.ap_profile_id,
|
||||
"to": [
|
||||
user.ap_profile_id
|
||||
],
|
||||
"object": {
|
||||
"actor": user.ap_profile_id,
|
||||
"to": None,
|
||||
"object": community.ap_profile_id,
|
||||
"type": "Follow",
|
||||
"id": follow_id
|
||||
},
|
||||
"type": "Accept",
|
||||
"id": f"https://{current_app.config['SERVER_NAME']}/activities/accept/" + gibberish(32)
|
||||
}
|
||||
try:
|
||||
HttpSignature.signed_request(user.ap_inbox_url, accept, community.private_key,
|
||||
f"https://{current_app.config['SERVER_NAME']}/c/{community.name}#main-key")
|
||||
except Exception as e:
|
||||
accept_log = ActivityPubLog(direction='out', activity_json=json.dumps(accept),
|
||||
result='failure', activity_id=accept['id'],
|
||||
exception_message = 'could not send Accept' + str(e))
|
||||
db.session.add(accept_log)
|
||||
db.session.commit()
|
||||
return
|
||||
activity_log.result = 'success'
|
||||
else:
|
||||
activity_log.result='failure'
|
||||
activity_log.exception_message = 'Could not detect type of like'
|
||||
if activity_log.result == 'success':
|
||||
... # todo: recalculate 'hotness' of liked post/reply
|
||||
|
||||
# remote user wants to follow one of our communities
|
||||
elif request_json['type'] == 'Follow':
|
||||
user_ap_id = request_json['actor']
|
||||
community_ap_id = request_json['object']
|
||||
follow_id = request_json['id']
|
||||
user = find_actor_or_create(user_ap_id)
|
||||
community = find_actor_or_create(community_ap_id)
|
||||
if user is not None and community is not None:
|
||||
# check if user is banned from this community
|
||||
banned = CommunityBan.query.filter_by(user_id=user.id,
|
||||
community_id=community.id).first()
|
||||
if banned is None:
|
||||
if not user.subscribed(community):
|
||||
activity_log.exception_message = 'user is banned from this community'
|
||||
# Accept: remote server is accepting our previous follow request
|
||||
elif request_json['type'] == 'Accept':
|
||||
if request_json['object']['type'] == 'Follow':
|
||||
community_ap_id = request_json['actor']
|
||||
user_ap_id = request_json['object']['actor']
|
||||
user = find_actor_or_create(user_ap_id)
|
||||
community = find_actor_or_create(community_ap_id)
|
||||
join_request = CommunityJoinRequest.query.filter_by(user_id=user.id,
|
||||
community_id=community.id).first()
|
||||
if join_request:
|
||||
member = CommunityMember(user_id=user.id, community_id=community.id)
|
||||
db.session.add(member)
|
||||
db.session.commit()
|
||||
# send accept message to acknowledge the follow
|
||||
accept = {
|
||||
"@context": [
|
||||
"https://www.w3.org/ns/activitystreams",
|
||||
"https://w3id.org/security/v1",
|
||||
],
|
||||
"actor": community.ap_profile_id,
|
||||
"to": [
|
||||
user.ap_profile_id
|
||||
],
|
||||
"object": {
|
||||
"actor": user.ap_profile_id,
|
||||
"to": None,
|
||||
"object": community.ap_profile_id,
|
||||
"type": "Follow",
|
||||
"id": follow_id
|
||||
},
|
||||
"type": "Accept",
|
||||
"id": f"https://{current_app.config['SERVER_NAME']}/activities/accept/" + gibberish(32)
|
||||
}
|
||||
try:
|
||||
HttpSignature.signed_request(user.ap_inbox_url, accept, community.private_key,
|
||||
f"https://{current_app.config['SERVER_NAME']}/c/{community.name}#main-key")
|
||||
except Exception as e:
|
||||
accept_log = ActivityPubLog(direction='out', activity_json=json.dumps(accept),
|
||||
result='failure', activity_id=accept['id'],
|
||||
exception_message = 'could not send Accept' + str(e))
|
||||
db.session.add(accept_log)
|
||||
db.session.commit()
|
||||
return
|
||||
activity_log.result = 'success'
|
||||
else:
|
||||
activity_log.exception_message = 'user is banned from this community'
|
||||
# remote server is accepting our previous follow request
|
||||
elif request_json['type'] == 'Accept':
|
||||
if request_json['object']['type'] == 'Follow':
|
||||
community_ap_id = request_json['actor']
|
||||
user_ap_id = request_json['object']['actor']
|
||||
user = find_actor_or_create(user_ap_id)
|
||||
community = find_actor_or_create(community_ap_id)
|
||||
join_request = CommunityJoinRequest.query.filter_by(user_id=user.id,
|
||||
community_id=community.id).first()
|
||||
if join_request:
|
||||
member = CommunityMember(user_id=user.id, community_id=community.id)
|
||||
db.session.add(member)
|
||||
db.session.commit()
|
||||
activity_log.result = 'success'
|
||||
|
||||
activity_log.result = 'success'
|
||||
else:
|
||||
activity_log.exception_message = 'Instance banned'
|
||||
else:
|
||||
activity_log.exception_message = 'Could not verify signature'
|
||||
else:
|
||||
activity_log.exception_message = 'Actor could not be found: ' + request_json['actor']
|
||||
|
||||
if activity_log.exception_message is not None:
|
||||
activity_log.result = 'failure'
|
||||
db.session.add(activity_log)
|
||||
db.session.commit()
|
||||
|
||||
|
|
@ -346,10 +420,7 @@ def community_outbox(actor):
|
|||
posts = community.posts.limit(50).all()
|
||||
|
||||
community_data = {
|
||||
"@context": [
|
||||
"https://www.w3.org/ns/activitystreams",
|
||||
"https://w3id.org/security/v1",
|
||||
],
|
||||
"@context": default_context(),
|
||||
"type": "OrderedCollection",
|
||||
"id": f"https://{current_app.config['SERVER_NAME']}/c/{actor}/outbox",
|
||||
"totalItems": len(posts),
|
||||
|
|
|
|||
|
|
@ -1,22 +1,21 @@
|
|||
import json
|
||||
import os
|
||||
from datetime import datetime
|
||||
from typing import Union
|
||||
from typing import Union, Tuple
|
||||
import markdown2
|
||||
from flask import current_app
|
||||
from sqlalchemy import text
|
||||
from app import db
|
||||
from app.models import User, Post, Community, BannedInstances, File
|
||||
from app import db, cache
|
||||
from app.models import User, Post, Community, BannedInstances, File, PostReply
|
||||
import time
|
||||
import base64
|
||||
import requests
|
||||
from cryptography.hazmat.primitives import serialization, hashes
|
||||
from cryptography.hazmat.primitives.asymmetric import padding
|
||||
from app.constants import *
|
||||
import functools
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from app.utils import get_request
|
||||
from app.utils import get_request, allowlist_html
|
||||
|
||||
|
||||
def public_key():
|
||||
|
|
@ -177,8 +176,11 @@ def banned_user_agents():
|
|||
return [] # todo: finish this function
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=100)
|
||||
def instance_blocked(host):
|
||||
@cache.cached(150)
|
||||
def instance_blocked(host: str) -> bool:
|
||||
host = host.lower()
|
||||
if 'https://' in host or 'http://' in host:
|
||||
host = urlparse(host).hostname
|
||||
instance = BannedInstances.query.filter_by(domain=host.strip()).first()
|
||||
return instance is not None
|
||||
|
||||
|
|
@ -198,7 +200,8 @@ def find_actor_or_create(actor: str) -> Union[User, Community, None]:
|
|||
server, address = extract_domain_and_actor(actor)
|
||||
if instance_blocked(server):
|
||||
return None
|
||||
user = User.query.filter_by(ap_profile_id=actor).first() # finds users formatted like https://kbin.social/u/tables
|
||||
user = User.query.filter_by(
|
||||
ap_profile_id=actor).first() # finds users formatted like https://kbin.social/u/tables
|
||||
if user is None:
|
||||
user = Community.query.filter_by(ap_profile_id=actor).first()
|
||||
if user is None:
|
||||
|
|
@ -301,6 +304,75 @@ def parse_summary(user_json) -> str:
|
|||
html_content = markdown2.markdown(markdown_text)
|
||||
return html_content
|
||||
elif 'summary' in user_json:
|
||||
return user_json['summary']
|
||||
return allowlist_html(user_json['summary'])
|
||||
else:
|
||||
return ''
|
||||
|
||||
|
||||
def default_context():
|
||||
context = [
|
||||
"https://www.w3.org/ns/activitystreams",
|
||||
"https://w3id.org/security/v1",
|
||||
]
|
||||
if current_app.config['FULL_AP_CONTEXT']:
|
||||
context.append({
|
||||
"lemmy": "https://join-lemmy.org/ns#",
|
||||
"litepub": "http://litepub.social/ns#",
|
||||
"pt": "https://joinpeertube.org/ns#",
|
||||
"sc": "http://schema.org/",
|
||||
"ChatMessage": "litepub:ChatMessage",
|
||||
"commentsEnabled": "pt:commentsEnabled",
|
||||
"sensitive": "as:sensitive",
|
||||
"matrixUserId": "lemmy:matrixUserId",
|
||||
"postingRestrictedToMods": "lemmy:postingRestrictedToMods",
|
||||
"removeData": "lemmy:removeData",
|
||||
"stickied": "lemmy:stickied",
|
||||
"moderators": {
|
||||
"@type": "@id",
|
||||
"@id": "lemmy:moderators"
|
||||
},
|
||||
"expires": "as:endTime",
|
||||
"distinguished": "lemmy:distinguished",
|
||||
"language": "sc:inLanguage",
|
||||
"identifier": "sc:identifier"
|
||||
})
|
||||
return context
|
||||
|
||||
|
||||
def find_reply_parent(in_reply_to: str) -> Tuple[int, int, int]:
|
||||
if 'comment' in in_reply_to:
|
||||
parent_comment = PostReply.get_by_ap_id(in_reply_to)
|
||||
parent_comment_id = parent_comment.id
|
||||
post_id = parent_comment.post_id
|
||||
root_id = parent_comment.root_id
|
||||
elif 'post' in in_reply_to:
|
||||
parent_comment_id = None
|
||||
post = Post.get_by_ap_id(in_reply_to)
|
||||
post_id = post.id
|
||||
root_id = None
|
||||
else:
|
||||
parent_comment_id = None
|
||||
root_id = None
|
||||
post_id = None
|
||||
post = Post.get_by_ap_id(in_reply_to)
|
||||
if post:
|
||||
post_id = post.id
|
||||
else:
|
||||
parent_comment = PostReply.get_by_ap_id(in_reply_to)
|
||||
if parent_comment:
|
||||
parent_comment_id = parent_comment.id
|
||||
post_id = parent_comment.post_id
|
||||
root_id = parent_comment.root_id
|
||||
|
||||
return post_id, parent_comment_id, root_id
|
||||
|
||||
|
||||
def find_liked_object(ap_id) -> Union[Post, PostReply, None]:
|
||||
post = Post.get_by_ap_id(ap_id)
|
||||
if post:
|
||||
return post
|
||||
else:
|
||||
post_reply = PostReply.get_by_ap_id(ap_id)
|
||||
if post_reply:
|
||||
return post_reply
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
from app.utils import render_template
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
from datetime import date, datetime, timedelta
|
||||
from flask import render_template, redirect, url_for, flash, request, make_response, session, Markup, current_app
|
||||
from flask import redirect, url_for, flash, request, make_response, session, Markup, current_app
|
||||
from werkzeug.urls import url_parse
|
||||
from flask_login import login_user, logout_user, current_user
|
||||
from flask_babel import _
|
||||
|
|
@ -10,6 +10,7 @@ from app.auth.util import random_token
|
|||
from app.models import User
|
||||
from app.auth.email import send_password_reset_email, send_welcome_email, send_verification_email
|
||||
from app.activitypub.signature import RsaKeys
|
||||
from app.utils import render_template
|
||||
|
||||
|
||||
@bp.route('/login', methods=['GET', 'POST'])
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
from datetime import date, datetime, timedelta
|
||||
from flask import render_template, redirect, url_for, flash, request, make_response, session, Markup, current_app, abort
|
||||
from flask import redirect, url_for, flash, request, make_response, session, Markup, current_app, abort
|
||||
from flask_login import login_user, logout_user, current_user
|
||||
from flask_babel import _
|
||||
from app import db
|
||||
|
|
@ -9,7 +9,7 @@ from app.community.util import search_for_community, community_url_exists
|
|||
from app.constants import SUBSCRIPTION_MEMBER, SUBSCRIPTION_OWNER
|
||||
from app.models import User, Community, CommunityMember, CommunityJoinRequest, CommunityBan
|
||||
from app.community import bp
|
||||
from app.utils import get_setting
|
||||
from app.utils import get_setting, render_template
|
||||
from sqlalchemy import or_
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -2,12 +2,13 @@ from datetime import datetime
|
|||
|
||||
from app import db
|
||||
from app.main import bp
|
||||
from flask import g, jsonify, render_template, flash, request
|
||||
from flask import g, jsonify, flash, request
|
||||
from flask_moment import moment
|
||||
from flask_login import current_user
|
||||
from flask_babel import _, get_locale
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy_searchable import search
|
||||
from app.utils import render_template
|
||||
|
||||
from app.models import Community, CommunityMember
|
||||
|
||||
|
|
|
|||
|
|
@ -231,6 +231,7 @@ class Post(db.Model):
|
|||
body = db.Column(db.Text)
|
||||
body_html = db.Column(db.Text)
|
||||
type = db.Column(db.Integer)
|
||||
comments_enabled = db.Column(db.Boolean, default=True)
|
||||
has_embed = db.Column(db.Boolean, default=False)
|
||||
reply_count = db.Column(db.Integer, default=0)
|
||||
score = db.Column(db.Integer, default=0, index=True)
|
||||
|
|
@ -256,6 +257,10 @@ class Post(db.Model):
|
|||
|
||||
image = db.relationship(File, foreign_keys=[image_id], cascade="all, delete")
|
||||
|
||||
@classmethod
|
||||
def get_by_ap_id(cls, ap_id):
|
||||
return cls.query.filter_by(ap_id=ap_id).first()
|
||||
|
||||
|
||||
class PostReply(db.Model):
|
||||
id = db.Column(db.Integer, primary_key=True)
|
||||
|
|
@ -267,6 +272,7 @@ class PostReply(db.Model):
|
|||
root_id = db.Column(db.Integer)
|
||||
body = db.Column(db.Text)
|
||||
body_html = db.Column(db.Text)
|
||||
body_html_safe = db.Column(db.Boolean, default=False)
|
||||
score = db.Column(db.Integer, default=0, index=True)
|
||||
nsfw = db.Column(db.Boolean, default=False)
|
||||
nsfl = db.Column(db.Boolean, default=False)
|
||||
|
|
@ -280,15 +286,21 @@ class PostReply(db.Model):
|
|||
edited_at = db.Column(db.DateTime)
|
||||
|
||||
ap_id = db.Column(db.String(255), index=True)
|
||||
ap_create_id = db.Column(db.String(100))
|
||||
ap_announce_id = db.Column(db.String(100))
|
||||
|
||||
search_vector = db.Column(TSVectorType('body'))
|
||||
|
||||
@classmethod
|
||||
def get_by_ap_id(cls, ap_id):
|
||||
return cls.query.filter_by(ap_id=ap_id).first()
|
||||
|
||||
|
||||
class Domain(db.Model):
|
||||
id = db.Column(db.Integer, primary_key=True)
|
||||
name = db.Column(db.String(255), index=True)
|
||||
post_count = db.Column(db.Integer, default=0)
|
||||
banned = db.Column(db.Boolean, default=False, index=True)
|
||||
banned = db.Column(db.Boolean, default=False, index=True) # Domains can be banned site-wide (by admin) or DomainBlock'ed by users
|
||||
|
||||
|
||||
class DomainBlock(db.Model):
|
||||
|
|
|
|||
|
|
@ -49,10 +49,12 @@
|
|||
<td>{{ community.post_count }}</td>
|
||||
<td>{{ community.post_reply_count }}</td>
|
||||
<td>{{ moment(community.last_active).fromNow(refresh=True) }}</td>
|
||||
<td>{% if current_user.subscribed(community) %}
|
||||
<a class="btn btn-primary btn-sm" href="/community/{{ community.link() }}/unsubscribe">Unsubscribe</a>
|
||||
{% else %}
|
||||
<a class="btn btn-primary btn-sm" href="/community/{{ community.link() }}/subscribe">Subscribe</a>
|
||||
<td>{% if current_user.is_authenticated %}
|
||||
{% if current_user.subscribed(community) %}
|
||||
<a class="btn btn-primary btn-sm" href="/community/{{ community.link() }}/unsubscribe">Unsubscribe</a>
|
||||
{% else %}
|
||||
<a class="btn btn-primary btn-sm" href="/community/{{ community.link() }}/subscribe">Subscribe</a>
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
</td>
|
||||
</tr>
|
||||
|
|
|
|||
95
app/utils.py
95
app/utils.py
|
|
@ -1,13 +1,26 @@
|
|||
import functools
|
||||
import random
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import flask
|
||||
from bs4 import BeautifulSoup
|
||||
import html as html_module
|
||||
import requests
|
||||
import os
|
||||
from flask import current_app, json
|
||||
from app import db
|
||||
from app.models import Settings
|
||||
from app import db, cache
|
||||
from app.models import Settings, Domain, Instance, BannedInstances
|
||||
|
||||
|
||||
# Flask's render_template function, with support for themes added
|
||||
def render_template(template_name: str, **context) -> str:
|
||||
theme = get_setting('theme', '')
|
||||
if theme != '':
|
||||
return flask.render_template(f'themes/{theme}/{template_name}', **context)
|
||||
else:
|
||||
return flask.render_template(template_name, **context)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Jinja: when a file was modified. Useful for cache-busting
|
||||
def getmtime(filename):
|
||||
return os.path.getmtime('static/' + filename)
|
||||
|
|
@ -28,8 +41,8 @@ def get_request(uri, params=None, headers=None) -> requests.Response:
|
|||
return response
|
||||
|
||||
|
||||
# saves an arbitrary object into a persistent key-value store. Possibly redis would be faster than using the DB
|
||||
@functools.lru_cache(maxsize=100)
|
||||
# saves an arbitrary object into a persistent key-value store. cached.
|
||||
@cache.cached(timeout=50)
|
||||
def get_setting(name: str, default=None):
|
||||
setting = Settings.query.filter_by(name=name).first()
|
||||
if setting is None:
|
||||
|
|
@ -46,7 +59,7 @@ def set_setting(name: str, value):
|
|||
else:
|
||||
setting.value = json.dumps(value)
|
||||
db.session.commit()
|
||||
get_setting.cache_clear()
|
||||
cache.delete_memoized(get_setting)
|
||||
|
||||
|
||||
# Return the contents of a file as a string. Inspired by PHP's function of the same name.
|
||||
|
|
@ -61,3 +74,73 @@ random_chars = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
|
|||
|
||||
def gibberish(length: int = 10) -> str:
|
||||
return "".join([random.choice(random_chars) for x in range(length)])
|
||||
|
||||
|
||||
def is_image_url(url):
|
||||
parsed_url = urlparse(url)
|
||||
path = parsed_url.path.lower()
|
||||
common_image_extensions = ['.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.webp']
|
||||
return any(path.endswith(extension) for extension in common_image_extensions)
|
||||
|
||||
|
||||
# sanitise HTML using an allow list
|
||||
def allowlist_html(html: str) -> str:
|
||||
allowed_tags = ['p', 'strong', 'a', 'ul', 'ol', 'li', 'em', 'blockquote', 'cite', 'br', 'h3', 'h4', 'h5']
|
||||
# Parse the HTML using BeautifulSoup
|
||||
soup = BeautifulSoup(html, 'html.parser')
|
||||
|
||||
# Find all tags in the parsed HTML
|
||||
for tag in soup.find_all():
|
||||
# If the tag is not in the allowed_tags list, remove it and its contents
|
||||
if tag.name not in allowed_tags:
|
||||
tag.extract()
|
||||
else:
|
||||
# Filter and sanitize attributes
|
||||
for attr in list(tag.attrs):
|
||||
if attr not in ['href', 'src']: # Add allowed attributes here
|
||||
del tag[attr]
|
||||
|
||||
# Encode the HTML to prevent script execution
|
||||
return html_module.escape(str(soup))
|
||||
|
||||
|
||||
# convert basic HTML to Markdown
|
||||
def html_to_markdown(html: str) -> str:
|
||||
soup = BeautifulSoup(html, 'html.parser')
|
||||
return html_to_markdown_worker(soup)
|
||||
|
||||
|
||||
def html_to_markdown_worker(element, indent_level=0):
|
||||
formatted_text = ''
|
||||
for item in element.contents:
|
||||
if isinstance(item, str):
|
||||
formatted_text += item
|
||||
elif item.name == 'p':
|
||||
formatted_text += '\n\n'
|
||||
elif item.name == 'br':
|
||||
formatted_text += ' \n' # Double space at the end for line break
|
||||
elif item.name == 'strong':
|
||||
formatted_text += '**' + html_to_markdown_worker(item) + '**'
|
||||
elif item.name == 'ul':
|
||||
formatted_text += '\n'
|
||||
formatted_text += html_to_markdown_worker(item, indent_level + 1)
|
||||
formatted_text += '\n'
|
||||
elif item.name == 'ol':
|
||||
formatted_text += '\n'
|
||||
formatted_text += html_to_markdown_worker(item, indent_level + 1)
|
||||
formatted_text += '\n'
|
||||
elif item.name == 'li':
|
||||
bullet = '-' if item.find_parent(['ul', 'ol']) and item.find_previous_sibling() is None else ''
|
||||
formatted_text += ' ' * indent_level + bullet + ' ' + html_to_markdown_worker(item).strip() + '\n'
|
||||
elif item.name == 'blockquote':
|
||||
formatted_text += ' ' * indent_level + '> ' + html_to_markdown_worker(item).strip() + '\n'
|
||||
elif item.name == 'code':
|
||||
formatted_text += '`' + html_to_markdown_worker(item) + '`'
|
||||
return formatted_text
|
||||
|
||||
|
||||
def domain_from_url(url: str) -> Domain:
|
||||
parsed_url = urlparse(url)
|
||||
domain = Domain.query.filter_by(name=parsed_url.hostname.lower()).first()
|
||||
return domain
|
||||
|
||||
|
|
|
|||
|
|
@ -21,3 +21,9 @@ class Config(object):
|
|||
RECAPTCHA3_PRIVATE_KEY = os.environ.get("RECAPTCHA3_PRIVATE_KEY")
|
||||
MODE = os.environ.get('MODE') or 'development'
|
||||
LANGUAGES = ['en']
|
||||
FULL_AP_CONTEXT = os.environ.get('FULL_AP_CONTEXT') or True
|
||||
CACHE_TYPE = os.environ.get('CACHE_TYPE') or 'FileSystemCache'
|
||||
CACHE_DIR = os.environ.get('CACHE_DIR') or '/dev/shm/pyfedi'
|
||||
CACHE_DEFAULT_TIMEOUT = 300
|
||||
CACHE_THRESHOLD = 1000
|
||||
CACHE_KEY_PREFIX = 'pyfedi'
|
||||
|
|
|
|||
|
|
@ -20,3 +20,5 @@ arrow==1.2.3
|
|||
pyld==2.0.3
|
||||
boto3==1.28.35
|
||||
markdown2==2.4.10
|
||||
beautifulsoup4==4.12.2
|
||||
flask-caching==2.0.2
|
||||
|
|
|
|||
Loading…
Reference in a new issue