move long-running tasks to separate background process (celery + redis)

.gitignore

@@ -161,3 +161,4 @@ cython_debug/
 .idea/
 app/static/*.css.map
 /app/static/media/
+celery_worker.py

app/__init__.py

@@ -13,6 +13,7 @@ from flask_mail import Mail
 from flask_moment import Moment
 from flask_babel import Babel, lazy_gettext as _l
 from flask_caching import Cache
+from celery import Celery
 from sqlalchemy_searchable import make_searchable
 
 from config import Config
@@ -28,6 +29,7 @@ bootstrap = Bootstrap5()
 moment = Moment()
 babel = Babel()
 cache = Cache()
+celery = Celery(__name__, broker=Config.CELERY_BROKER_URL)
 
 
 def create_app(config_class=Config):
@@ -43,6 +45,7 @@ def create_app(config_class=Config):
     make_searchable(db.metadata)
     babel.init_app(app, locale_selector=get_locale)
     cache.init_app(app)
+    celery.conf.update(app.config)
 
     from app.main import bp as main_bp
     app.register_blueprint(main_bp)

app/activitypub/util.py

@@ -215,7 +215,6 @@ def find_actor_or_create(actor: str) -> Union[User, Community, None]:
                     if 'rel' in links and links['rel'] == 'self':  # this contains the URL of the activitypub profile
                         type = links['type'] if 'type' in links else 'application/activity+json'
                         # retrieve the activitypub profile
-                        print('****', links['href'])
                         actor_data = get_request(links['href'], headers={'Accept': type})
                         # to see the structure of the json contained in actor_data, do a GET to https://lemmy.world/c/technology with header Accept: application/activity+json
                         if actor_data.status_code == 200:
@@ -462,7 +461,10 @@ def find_instance_id(server):
         try:
             instance_data = get_request(f"https://{server}", headers={'Accept': 'application/activity+json'})
         except:
-             return None
+            new_instance = Instance(domain=server, software='unknown', created_at=utcnow())
+            db.session.add(new_instance)
+            db.session.commit()
+            return new_instance.id
         if instance_data.status_code == 200:
             try:
                 instance_json = instance_data.json()
@@ -487,7 +489,11 @@ def find_instance_id(server):
             db.session.add(new_instance)
             db.session.commit()
             return new_instance.id
-    return None
+        else:
+            new_instance = Instance(domain=server, software='unknown', created_at=utcnow())
+            db.session.add(new_instance)
+            db.session.commit()
+            return new_instance.id
 
 
 # alter the effect of upvotes based on their instance. Default to 1.0

app/email.py

@@ -1,5 +1,5 @@
 from flask import current_app, render_template, escape
-from app import db
+from app import db, celery
 from flask_babel import _, lazy_gettext as _l           # todo: set the locale based on account_id so that _() works
 import boto3
 from botocore.exceptions import ClientError
@@ -9,6 +9,7 @@ AWS_REGION = "ap-southeast-2"
 CHARSET = "UTF-8"
 
 
+@celery.task
 def send_async_email(subject, sender, recipients, text_body, html_body, reply_to):
     if type(recipients) == str:
         recipients = [recipients]
@@ -62,5 +63,7 @@ def send_async_email(subject, sender, recipients, text_body, html_body, reply_to
 
 
 def send_email(subject, sender, recipients: List[str], text_body, html_body, reply_to=None):
-    # todo: make async or threaded
+    if current_app.debug:
-    send_async_email(subject, sender, recipients, text_body, html_body, reply_to)
+        send_async_email(subject, sender, recipients, text_body, html_body, reply_to)
+    else:
+        send_async_email.delay(subject, sender, recipients, text_body, html_body, reply_to)

app/main/routes.py

@@ -107,6 +107,7 @@ def verification_warning():
         flash(_('Please click the link in your email inbox to verify your account.'), 'warning')
 
 
+@cache.cached(timeout=6)
 def activitypub_application():
     application_data = {
         '@context': default_context(),

app/static/structure.css

@@ -488,6 +488,16 @@ fieldset legend {
   max-width: 100%;
 }
 
+.render_username {
+  display: inline;
+}
+.render_username a img {
+  width: 20px;
+  height: 20px;
+  border-radius: 50%;
+  vertical-align: bottom;
+}
+
 .comments > .comment {
   margin-left: 0;
   border-top: solid 1px #bbb;

app/static/structure.scss

@@ -198,6 +198,16 @@ nav, etc which are used site-wide */
   }
 }
 
+.render_username {
+  display: inline;
+  a img {
+    width: 20px;
+    height: 20px;
+    border-radius: 50%;
+    vertical-align: bottom;
+  }
+}
+
 .comments > .comment {
   margin-left: 0;
   border-top: solid 1px $grey;

app/templates/base.html

@@ -1,12 +1,18 @@
 {% macro render_username(user) %}
-{% if user.deleted %}
+<span class="render_username">
-    [deleted]
+    {% if user.deleted %}
-{% else %}
+        [deleted]
-    <a href="/u/{{ user.link() }}" title="{{ user.ap_id if user.ap_id != none else user.user_name }}">{{ user.user_name }}</a>
+    {% else %}
-    {% if user.created_recently() %}
+        {% if user.avatar_id %}
-        <span class="fe fe-new-account" title="New account"> </span>
+            <a href="/u/{{ user.link() }}" title="{{ user.ap_id if user.ap_id != none else user.user_name }}">
+                <img src="{{ user.avatar_image() }}" alt="Avatar" /></a>
+        {% endif %}
+        <a href="/u/{{ user.link() }}" title="{{ user.ap_id if user.ap_id != none else user.user_name }}">{{ user.user_name }}</a>
+        {% if user.created_recently() %}
+            <span class="fe fe-new-account" title="New account"> </span>
+        {% endif %}
     {% endif %}
-{% endif %}
+</span>
 {% endmacro %}
 <!doctype html>
 <html lang="en">

app/templates/community/community.html

@@ -84,8 +84,8 @@
                  <h2>{{ _('About community') }}</h2>
             </div>
             <div class="card-body">
-                <p>{{ community.description|safe }}</p>
+                <p>{{ community.description_html|safe }}</p>
-                <p>{{ community.rules|safe }}</p>
+                <p>{{ community.rules_html|safe }}</p>
                 {% if len(mods) > 0 and not community.private_mods %}
                     <h3>Moderators</h3>
                     <ul>

app/templates/post/post.html

@@ -134,8 +134,8 @@
                  <h2>{{ _('About community') }}</h2>
             </div>
             <div class="card-body">
-                <p>{{ post.community.description|safe }}</p>
+                <p>{{ post.community.description_html|safe }}</p>
-                <p>{{ post.community.rules|safe }}</p>
+                <p>{{ post.community.rules_html|safe }}</p>
                 {% if len(mods) > 0 and not post.community.private_mods %}
                     <h3>Moderators</h3>
                     <ol>

app/utils.py

@@ -42,7 +42,6 @@ def render_template(template_name: str, **context) -> Response:
 
 
 def request_etag_matches(etag):
-    print(str(request.headers))
     if 'If-None-Match' in request.headers:
         old_etag = request.headers['If-None-Match']
         return old_etag == etag

celery_worker.default.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+import os
+from app import celery, create_app
+
+
+app = create_app()
+if not app.debug:
+    os.environ['DATABASE_URL'] = 'postgresql+psycopg2://pyfedi:pyfedi@127.0.0.1/pyfedi'
+    os.environ['SERVER_NAME'] = 'piefed.ngrok.app'
+
+app.app_context().push()

config.py

@@ -27,4 +27,6 @@ class Config(object):
     CACHE_DEFAULT_TIMEOUT = 300
     CACHE_THRESHOLD = 1000
     CACHE_KEY_PREFIX = 'pyfedi'
+    CELERY_BROKER_URL = os.environ.get('CELERY_BROKER_URL') or 'redis://localhost:6379/0'
+    RESULT_BACKEND = os.environ.get('RESULT_BACKEND') or 'redis://localhost:6379/0'
     SQLALCHEMY_ECHO = False     # set to true to see SQL in console

dev_notes.txt

@@ -0,0 +1,12 @@
+for celery, run this:
+
+celery -A celery_worker.celery worker --loglevel=INFO
+
+
+on prod web server, celery is managed by systemd: /etc/default/celeryd and /etc/systemd/system/celeryd.service
+
+sudo systemctl stop celeryd
+sudo systemctl restart celeryd or sudo service celeryd restart
+
+*** check for celery-related problems by looking in /var/log/celery ***
+

interests.txt

@@ -144,7 +144,6 @@ environment
     https://sh.itjust.works/c/sewingrepairing
     https://lemmy.world/c/fuckcars
     https://lemmy.world/c/evs
-    https://feddit.uk/c/evs
     https://slrpnk.net/c/solarpunk
     https://slrpnk.net/c/climate
     https://slrpnk.net/c/energy

requirements.txt

@@ -26,3 +26,5 @@ Pillow
 pillow-heif
 opengraph-parse=0.0.6
 feedgen==0.9.0
+celery==5.3.6
+redis==5.0.1