diff --git a/app/activitypub/routes.py b/app/activitypub/routes.py index 5af7eadb..b1668107 100644 --- a/app/activitypub/routes.py +++ b/app/activitypub/routes.py @@ -13,7 +13,7 @@ from app.models import User, Community, CommunityJoinRequest, CommunityMember, C PostReply, Instance, PostVote, PostReplyVote, File, AllowedInstances, BannedInstances from app.activitypub.util import public_key, users_total, active_half_year, active_month, local_posts, local_comments, \ post_to_activity, find_actor_or_create, default_context, instance_blocked, find_reply_parent, find_liked_object, \ - lemmy_site_data, instance_weight + lemmy_site_data, instance_weight, cache_key_by_ap_header, is_activitypub_request from app.utils import gibberish, get_setting, is_image_url, allowlist_html, html_to_markdown, render_template, \ domain_from_url, markdown_to_html, community_membership, ap_datetime import werkzeug.exceptions @@ -69,6 +69,7 @@ def webfinger(): @bp.route('/.well-known/nodeinfo') +@cache.cached(timeout=600) def nodeinfo(): nodeinfo_data = {"links": [{"rel": "http://nodeinfo.diaspora.software/ns/schema/2.0", "href": f"https://{current_app.config['SERVER_NAME']}/nodeinfo/2.0"}]} @@ -77,6 +78,7 @@ def nodeinfo(): @bp.route('/nodeinfo/2.0') @bp.route('/nodeinfo/2.0.json') +@cache.cached(timeout=600) def nodeinfo2(): nodeinfo_data = { @@ -103,11 +105,13 @@ def nodeinfo2(): @bp.route('/api/v3/site') +@cache.cached(timeout=600) def lemmy_site(): return jsonify(lemmy_site_data()) @bp.route('/api/v3/federated_instances') +@cache.cached(timeout=600) def lemmy_federated_instances(): instances = Instance.query.all() linked = [] @@ -133,11 +137,8 @@ def lemmy_federated_instances(): }) -def is_activitypub_request(): - return 'application/ld+json' in request.headers.get('Accept', '') or 'application/activity+json' in request.headers.get('Accept', '') - - @bp.route('/u/', methods=['GET']) +@cache.cached(timeout=10, make_cache_key=cache_key_by_ap_header) def user_profile(actor): """ Requests to this endpoint can be for a JSON representation of the user, or a HTML rendering of their profile. The two types of requests are differentiated by the header """ @@ -290,7 +291,8 @@ def shared_inbox(): community = find_actor_or_create(community_ap_id) user = find_actor_or_create(user_ap_id) if user and community: - user.last_seen = datetime.utcnow() + user.last_seen = community.last_active = datetime.utcnow() + object_type = request_json['object']['type'] new_content_types = ['Page', 'Article', 'Link', 'Note'] if object_type in new_content_types: # create a new post @@ -399,7 +401,7 @@ def shared_inbox(): community = find_actor_or_create(community_ap_id) user = find_actor_or_create(user_ap_id) if user and community: - user.last_seen = datetime.utcnow() + user.last_seen = community.last_active = datetime.utcnow() object_type = request_json['object']['object']['type'] new_content_types = ['Page', 'Article', 'Link', 'Note'] if object_type in new_content_types: # create a new post @@ -853,6 +855,13 @@ def shared_inbox(): activity_log.result = 'success' else: activity_log.exception_message = 'Could not find user or content for vote' + # Flush the caches of any major object that was created. To be sure. + if 'user' in vars() and user is not None: + user.flush_cache() + #if 'community' in vars() and community is not None: + # community.flush_cache() + if 'post' in vars() and post is not None: + post.flush_cache() else: activity_log.exception_message = 'Instance banned' else: @@ -889,6 +898,7 @@ def community_outbox(actor): @bp.route('/c//moderators', methods=['GET']) +@cache.cached(timeout=10, make_cache_key=cache_key_by_ap_header) def community_moderators(actor): actor = actor.strip() community = Community.query.filter_by(name=actor, banned=False, ap_id=None).first() @@ -935,6 +945,7 @@ def inbox(actor): @bp.route('/comment/', methods=['GET']) +@cache.cached(timeout=10, make_cache_key=cache_key_by_ap_header) def comment_ap(comment_id): if is_activitypub_request(): reply = PostReply.query.get_or_404(comment_id) @@ -974,6 +985,7 @@ def comment_ap(comment_id): @bp.route('/post/', methods=['GET', 'POST']) +@cache.cached(timeout=10, make_cache_key=cache_key_by_ap_header) def post_ap(post_id): if request.method == 'GET' and is_activitypub_request(): post = Post.query.get_or_404(post_id) diff --git a/app/activitypub/util.py b/app/activitypub/util.py index fb008b45..42758e13 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -2,7 +2,7 @@ import json import os from datetime import datetime from typing import Union, Tuple -from flask import current_app +from flask import current_app, request from sqlalchemy import text from app import db, cache from app.models import User, Post, Community, BannedInstances, File, PostReply, AllowedInstances, Instance @@ -405,6 +405,15 @@ def instance_weight(domain): return 1.0 +def is_activitypub_request(): + return 'application/ld+json' in request.headers.get('Accept', '') or 'application/activity+json' in request.headers.get('Accept', '') + + +# differentiate between cached JSON and cached HTML by appending is_activitypub_request() to the cache key +def cache_key_by_ap_header(**kwargs): + return request.path + "_" + str(is_activitypub_request()) + + def lemmy_site_data(): data = { "site_view": { diff --git a/app/community/routes.py b/app/community/routes.py index c6d34c03..980ea9b8 100644 --- a/app/community/routes.py +++ b/app/community/routes.py @@ -16,7 +16,8 @@ from app.models import User, Community, CommunityMember, CommunityJoinRequest, C File, PostVote from app.community import bp from app.utils import get_setting, render_template, allowlist_html, markdown_to_html, validation_required, \ - shorten_string, markdown_to_text, domain_from_url, validate_image, gibberish, community_membership, ap_datetime + shorten_string, markdown_to_text, domain_from_url, validate_image, gibberish, community_membership, ap_datetime, \ + request_etag_matches, return_304 import os from PIL import Image, ImageOps from datetime import datetime @@ -87,6 +88,12 @@ def add_remote(): # @bp.route('/c/', methods=['GET']) - defined in activitypub/routes.py, which calls this function for user requests. A bit weird. def show_community(community: Community): + + # If nothing has changed since their last visit, return HTTP 304 + current_etag = f"{community.id}_{hash(community.last_active)}" + if request_etag_matches(current_etag): + return return_304(current_etag) + mods = community.moderators() is_moderator = current_user.is_authenticated and any(mod.user_id == current_user.id for mod in mods) @@ -110,7 +117,7 @@ def show_community(community: Community): return render_template('community/community.html', community=community, title=community.title, is_moderator=is_moderator, is_owner=is_owner, mods=mod_list, posts=posts, description=description, og_image=og_image, POST_TYPE_IMAGE=POST_TYPE_IMAGE, POST_TYPE_LINK=POST_TYPE_LINK, SUBSCRIPTION_PENDING=SUBSCRIPTION_PENDING, - SUBSCRIPTION_MEMBER=SUBSCRIPTION_MEMBER) + SUBSCRIPTION_MEMBER=SUBSCRIPTION_MEMBER, etag=f"{community.id}_{hash(community.last_active)}") @bp.route('//subscribe', methods=['GET']) diff --git a/app/models.py b/app/models.py index c1785b04..5905881e 100644 --- a/app/models.py +++ b/app/models.py @@ -355,8 +355,6 @@ class User(UserMixin, db.Model): def profile_id(self): return self.ap_profile_id if self.ap_profile_id else f"https://{current_app.config['SERVER_NAME']}/u/{self.user_name}" - - def created_recently(self): return self.created and self.created > datetime.utcnow() - timedelta(days=7) @@ -369,6 +367,10 @@ class User(UserMixin, db.Model): return return User.query.get(id) + def flush_cache(self): + cache.delete('/u/' + self.user_name + '_False') + cache.delete('/u/' + self.user_name + '_True') + def purge_content(self): files = File.query.join(Post).filter(Post.user_id == self.id).all() for file in files: @@ -446,6 +448,7 @@ class Post(db.Model): image = db.relationship(File, lazy='joined', foreign_keys=[image_id], cascade="all, delete") domain = db.relationship('Domain', lazy='joined', foreign_keys=[domain_id]) author = db.relationship('User', lazy='joined', overlaps='posts', foreign_keys=[user_id]) + replies = db.relationship('PostReply', lazy='dynamic', backref='post') def is_local(self): return self.ap_id is None or self.ap_id.startswith('https://' + current_app.config['SERVER_NAME']) @@ -472,6 +475,10 @@ class Post(db.Model): def profile_id(self): return f"https://{current_app.config['SERVER_NAME']}/post/{self.id}" + def flush_cache(self): + cache.delete(f'/post/{self.id}_False') + cache.delete(f'/post/{self.id}_True') + class PostReply(db.Model): query_class = FullTextSearchQuery @@ -515,7 +522,10 @@ class PostReply(db.Model): return cls.query.filter_by(ap_id=ap_id).first() def profile_id(self): - return f"https://{current_app.config['SERVER_NAME']}/comment/{self.id}" + if self.ap_id: + return self.ap_id + else: + return f"https://{current_app.config['SERVER_NAME']}/comment/{self.id}" # the ap_id of the parent object, whether it's another PostReply or a Post def in_reply_to(self): diff --git a/app/post/routes.py b/app/post/routes.py index af80f808..106265f5 100644 --- a/app/post/routes.py +++ b/app/post/routes.py @@ -1,12 +1,13 @@ from datetime import datetime -from flask import redirect, url_for, flash, request, make_response, session, Markup, current_app, abort +from flask import redirect, url_for, flash, current_app, abort from flask_login import login_user, logout_user, current_user, login_required from flask_babel import _ from sqlalchemy import or_, desc from app import db, constants from app.activitypub.signature import HttpSignature +from app.activitypub.util import default_context from app.community.util import save_post from app.post.forms import NewReplyForm from app.community.forms import CreatePostForm @@ -16,11 +17,18 @@ from app.models import Post, PostReply, \ PostReplyVote, PostVote, Notification from app.post import bp from app.utils import get_setting, render_template, allowlist_html, markdown_to_html, validation_required, \ - shorten_string, markdown_to_text, domain_from_url, validate_image, gibberish, ap_datetime + shorten_string, markdown_to_text, domain_from_url, validate_image, gibberish, ap_datetime, return_304, \ + request_etag_matches def show_post(post_id: int): post = Post.query.get_or_404(post_id) + + # If nothing has changed since their last visit, return HTTP 304 + current_etag = f"{post.id}_{hash(post.last_active)}" + if request_etag_matches(current_etag): + return return_304(current_etag) + mods = post.community.moderators() is_moderator = current_user.is_authenticated and any(mod.user_id == current_user.id for mod in mods) @@ -33,11 +41,12 @@ def show_post(post_id: int): notify_author=form.notify_author.data) if post.notify_author and current_user.id != post.user_id: # todo: check if replier is blocked notification = Notification(title=_('Reply: ') + shorten_string(form.body.data), user_id=post.user_id, - author_id=current_user.id, url=url_for('post.show_post', post_id=post.id)) + author_id=current_user.id, url=url_for('activitypub.post_ap', post_id=post.id)) db.session.add(notification) post.last_active = post.community.last_active = datetime.utcnow() post.reply_count += 1 post.community.post_reply_count += 1 + db.session.add(reply) db.session.commit() reply.ap_id = reply.profile_id() @@ -47,7 +56,9 @@ def show_post(post_id: int): db.session.commit() form.body.data = '' flash('Your comment has been added.') - # todo: flush cache + + post.flush_cache() + # federation if not post.community.is_local(): # this is a remote community, send it to the instance that hosts it reply_json = { @@ -100,7 +111,7 @@ def show_post(post_id: int): else: # local community - send it to followers on remote instances ... - return redirect(url_for('post.show_post', + return redirect(url_for('activitypub.post_ap', post_id=post_id)) # redirect to current page to avoid refresh resubmitting the form else: replies = post_replies(post.id, 'top') @@ -112,7 +123,8 @@ def show_post(post_id: int): return render_template('post/post.html', title=post.title, post=post, is_moderator=is_moderator, canonical=post.ap_id, form=form, replies=replies, THREAD_CUTOFF_DEPTH=constants.THREAD_CUTOFF_DEPTH, description=description, og_image=og_image, POST_TYPE_IMAGE=constants.POST_TYPE_IMAGE, - POST_TYPE_LINK=constants.POST_TYPE_LINK, POST_TYPE_ARTICLE=constants.POST_TYPE_ARTICLE) + POST_TYPE_LINK=constants.POST_TYPE_LINK, POST_TYPE_ARTICLE=constants.POST_TYPE_ARTICLE, + etag=f"{post.id}_{hash(post.last_active)}") @bp.route('/post//', methods=['GET', 'POST']) @@ -163,6 +175,7 @@ def post_vote(post_id: int, vote_direction): db.session.add(vote) current_user.last_seen = datetime.utcnow() db.session.commit() + post.flush_cache() return render_template('post/_post_voting_buttons.html', post=post, upvoted_class=upvoted_class, downvoted_class=downvoted_class) @@ -214,6 +227,7 @@ def comment_vote(comment_id, vote_direction): db.session.add(vote) current_user.last_seen = datetime.utcnow() db.session.commit() + comment.post.flush_cache() return render_template('post/_voting_buttons.html', comment=comment, upvoted_class=upvoted_class, downvoted_class=downvoted_class) @@ -249,7 +263,7 @@ def add_reply(post_id: int, comment_id: int): db.session.add(reply) if in_reply_to.notify_author and current_user.id != in_reply_to.user_id and in_reply_to.author.ap_id is None: # todo: check if replier is blocked notification = Notification(title=_('Reply: ') + shorten_string(form.body.data), user_id=in_reply_to.user_id, - author_id=current_user.id, url=url_for('post.show_post', post_id=post.id)) + author_id=current_user.id, url=url_for('activitypub.post_ap', post_id=post.id)) db.session.add(notification) db.session.commit() reply.ap_id = reply.profile_id() @@ -262,7 +276,8 @@ def add_reply(post_id: int, comment_id: int): db.session.commit() form.body.data = '' flash('Your comment has been added.') - # todo: flush cache + + post.flush_cache() # federation if not post.community.is_local(): # this is a remote community, send it to the instance that hosts it @@ -271,13 +286,16 @@ def add_reply(post_id: int, comment_id: int): 'id': reply.profile_id(), 'attributedTo': current_user.profile_id(), 'to': [ - 'https://www.w3.org/ns/activitystreams#Public' + 'https://www.w3.org/ns/activitystreams#Public', + in_reply_to.author.profile_id() ], 'cc': [ post.community.profile_id(), + current_user.followers_url() ], 'content': reply.body_html, 'inReplyTo': in_reply_to.profile_id(), + 'url': reply.profile_id(), 'mediaType': 'text/html', 'source': { 'content': reply.body, @@ -285,31 +303,28 @@ def add_reply(post_id: int, comment_id: int): }, 'published': ap_datetime(datetime.utcnow()), 'distinguished': False, - 'audience': post.community.profile_id() + 'audience': post.community.profile_id(), + 'contentMap': { + 'en': reply.body_html + } } create_json = { + '@context': default_context(), 'type': 'Create', 'actor': current_user.profile_id(), 'audience': post.community.profile_id(), 'to': [ - 'https://www.w3.org/ns/activitystreams#Public' + 'https://www.w3.org/ns/activitystreams#Public', + in_reply_to.author.profile_id() ], 'cc': [ - post.community.ap_profile_id + post.community.profile_id(), + current_user.followers_url() ], 'object': reply_json, 'id': f"https://{current_app.config['SERVER_NAME']}/activities/create/{gibberish(15)}" } if in_reply_to.notify_author and in_reply_to.author.ap_id is not None: - create_json['cc'].append(in_reply_to.author.ap_profile_id) - create_json['tag'] = [ - { - 'href': in_reply_to.author.ap_profile_id, - 'name': '@' + in_reply_to.author.ap_id, - 'type': 'Mention' - } - ] - reply_json['cc'].append(in_reply_to.author.ap_profile_id) reply_json['tag'] = [ { 'href': in_reply_to.author.ap_profile_id, @@ -327,12 +342,12 @@ def add_reply(post_id: int, comment_id: int): current_app.logger.error('Response code for reply attempt was ' + str(message.status_code) + ' ' + message.text) except Exception as ex: - flash('Failed to send request to subscribe: ' + str(ex), 'error') - current_app.logger.error("Exception while trying to subscribe" + str(ex)) + flash('Failed to send reply: ' + str(ex), 'error') + current_app.logger.error("Exception while trying to send reply" + str(ex)) else: # local community - send it to followers on remote instances ... if reply.depth <= constants.THREAD_CUTOFF_DEPTH: - return redirect(url_for('post.show_post', post_id=post_id, _anchor=f'comment_{reply.parent_id}')) + return redirect(url_for('activitypub.post_ap', post_id=post_id, _anchor=f'comment_{reply.parent_id}')) else: return redirect(url_for('post.continue_discussion', post_id=post_id, comment_id=reply.parent_id)) else: @@ -365,8 +380,9 @@ def post_edit(post_id: int): post.community.last_active = datetime.utcnow() post.edited_at = datetime.utcnow() db.session.commit() + post.flush_cache() flash(_('Your changes have been saved.'), 'success') - return redirect(url_for('post.show_post', post_id=post.id)) + return redirect(url_for('activitypub.post_ap', post_id=post.id)) else: if post.type == constants.POST_TYPE_ARTICLE: form.type.data = 'discussion' @@ -392,6 +408,7 @@ def post_delete(post_id: int): community = post.community if post.user_id == current_user.id or community.is_moderator(): post.delete_dependencies() + post.flush_cache() db.session.delete(post) db.session.commit() flash('Post deleted.') diff --git a/app/templates/post/_post_teaser.html b/app/templates/post/_post_teaser.html index 952077ea..b3067a34 100644 --- a/app/templates/post/_post_teaser.html +++ b/app/templates/post/_post_teaser.html @@ -4,7 +4,7 @@

- {{ post.title }} + {{ post.title }} {% if post.type == POST_TYPE_IMAGE %} {% endif %} {% if post.type == POST_TYPE_LINK and post.domain_id %} {% if post.url and 'youtube.com' in post.url %} @@ -19,7 +19,7 @@ {{ render_username(post.author) }} ยท {{ moment(post.posted_at).fromNow() }} {% if post.image_id %}
- {{ post.image.alt_text }}{{ post.image.alt_text }}
{% endif %} @@ -28,8 +28,8 @@

- - {{ post.reply_count }} + + {{ post.reply_count }}
diff --git a/app/templates/post/continue_discussion.html b/app/templates/post/continue_discussion.html index f72fcd2d..e93d0b50 100644 --- a/app/templates/post/continue_discussion.html +++ b/app/templates/post/continue_discussion.html @@ -5,7 +5,7 @@
{% include 'post/_post_full.html' %} -

Back to main discussion

+

Back to main discussion

{% macro render_comment(comment) %} diff --git a/app/templates/user/notifications.html b/app/templates/user/notifications.html index 2f2b9014..4a5b4795 100644 --- a/app/templates/user/notifications.html +++ b/app/templates/user/notifications.html @@ -93,7 +93,7 @@
diff --git a/app/templates/user/show_profile.html b/app/templates/user/show_profile.html index faa7d2c3..fe1eb24c 100644 --- a/app/templates/user/show_profile.html +++ b/app/templates/user/show_profile.html @@ -143,7 +143,7 @@ diff --git a/app/user/routes.py b/app/user/routes.py index e4ff2ab5..1d79c70c 100644 --- a/app/user/routes.py +++ b/app/user/routes.py @@ -4,7 +4,7 @@ from flask import redirect, url_for, flash, request, make_response, session, Mar from flask_login import login_user, logout_user, current_user, login_required from flask_babel import _ -from app import db +from app import db, cache from app.models import Post, Community, CommunityMember, User, PostReply, PostVote, Notification from app.user import bp from app.user.forms import ProfileForm, SettingsForm @@ -47,9 +47,11 @@ def edit_profile(actor): if form.password_field.data.strip() != '': current_user.set_password(form.password_field.data) current_user.about = form.about.data + current_user.flush_cache() db.session.commit() flash(_('Your changes have been saved.'), 'success') + return redirect(url_for('user.edit_profile', actor=actor)) elif request.method == 'GET': form.email.data = current_user.email diff --git a/app/utils.py b/app/utils.py index 72badc29..d39e27f0 100644 --- a/app/utils.py +++ b/app/utils.py @@ -11,7 +11,7 @@ from bs4 import BeautifulSoup import requests import os import imghdr -from flask import current_app, json, redirect, url_for, request +from flask import current_app, json, redirect, url_for, request, make_response, Response from flask_login import current_user from sqlalchemy import text @@ -20,12 +20,33 @@ from app.models import Settings, Domain, Instance, BannedInstances, User, Commun # Flask's render_template function, with support for themes added -def render_template(template_name: str, **context) -> str: +def render_template(template_name: str, **context) -> Response: theme = get_setting('theme', '') if theme != '': - return flask.render_template(f'themes/{theme}/{template_name}', **context) + content = flask.render_template(f'themes/{theme}/{template_name}', **context) else: - return flask.render_template(template_name, **context) + content = flask.render_template(template_name, **context) + + # Browser caching using ETags and Cache-Control + resp = make_response(content) + if 'etag' in context: + resp.headers.add_header('ETag', context['etag']) + resp.headers.add_header('Cache-Control', 'no-cache, max-age=600, must-revalidate') + return resp + + +def request_etag_matches(etag): + if 'If-None-Match' in request.headers: + old_etag = request.headers['If-None-Match'] + return old_etag == etag + return False + + +def return_304(etag): + resp = make_response('', 304) + resp.headers.add_header('ETag', request.headers['If-None-Match']) + resp.headers.add_header('Cache-Control', 'no-cache, max-age=600, must-revalidate') + return resp # Jinja: when a file was modified. Useful for cache-busting