Merge branch 'main' of https://codeberg.org/saint/pyfedi

INSTALL.md

@@ -387,3 +387,31 @@ Once a week or so it's good to run remove_orphan_files.sh to save disk space:
 5 4 * * 1 rimu cd /home/rimu/pyfedi && /home/rimu/pyfedi/remove_orphan_files.sh
 ```
 
+---
+
+
+Email
+---
+
+Email can be sent either through SMTP or Amazon web services (SES). SES is faster but PieFed does not send much
+email so it probably doesn't matter which method you choose.
+
+### AWS SES
+
+PieFed uses Amazon's "boto3" module to connect to SES. Boto3 needs to log into AWS and that can be set up using a file
+at ~/.aws/credentials or environment variables. Details at https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html.
+
+In your .env you need to set the AWS region you're using for SES. Something like AWS_REGION = 'ap-southeast-2'.
+
+### SMTP
+
+To use SMTP you need to set all the MAIL_* environment variables in you .env file. See env.sample for a list of them.
+
+### Testing email
+
+You need to set MAIL_FROM in .env to some email address.
+
+Log into Piefed then go to https://yourdomain/test_email to trigger a test email. It will use SES or SMTP depending on
+which environment variables you defined in .env. If MAIL_SERVER is empty it will try SES. Then if AWS_REGION is empty it'll
+silently do nothing.
+

app/activitypub/util.py

@@ -1222,6 +1222,9 @@ def create_post(activity_log: ActivityPubLog, community: Community, request_json
             post.url = request_json['object']['attachment'][0]['href']
             if is_image_url(post.url):
                 post.type = POST_TYPE_IMAGE
+                image = File(source_url=request_json['object']['image']['url'])
+                db.session.add(image)
+                post.image = image
             else:
                 post.type = POST_TYPE_LINK
             domain = domain_from_url(post.url)
@@ -1247,7 +1250,7 @@ def create_post(activity_log: ActivityPubLog, community: Community, request_json
             if not domain.banned:
                 domain.post_count += 1
                 post.domain = domain
-    if 'image' in request_json['object']:
+    if 'image' in request_json['object'] and post.image is None:
         image = File(source_url=request_json['object']['image']['url'])
         db.session.add(image)
         post.image = image

app/admin/forms.py

@@ -95,7 +95,7 @@ class EditCommunityForm(FlaskForm):
 class EditTopicForm(FlaskForm):
     name = StringField(_l('Name'), validators=[DataRequired()])
     machine_name = StringField(_l('Url'), validators=[DataRequired()])
-    add_community = SelectField(_l('Community to add'), coerce=int, validators=[Optional()])
+    parent_id = SelectField(_l('Parent topic'), coerce=int, validators=[Optional()])
     submit = SubmitField(_l('Save'))
 
 

app/admin/routes.py

@@ -12,7 +12,8 @@ from app.activitypub.signature import post_request
 from app.activitypub.util import default_context
 from app.admin.forms import FederationForm, SiteMiscForm, SiteProfileForm, EditCommunityForm, EditUserForm, \
     EditTopicForm, SendNewsletterForm, AddUserForm
-from app.admin.util import unsubscribe_from_everything_then_delete, unsubscribe_from_community, send_newsletter
+from app.admin.util import unsubscribe_from_everything_then_delete, unsubscribe_from_community, send_newsletter, \
+    topic_tree, topics_for_form
 from app.community.util import save_icon_file, save_banner_file
 from app.models import AllowedInstances, BannedInstances, ActivityPubLog, utcnow, Site, Community, CommunityMember, \
     User, Instance, File, Report, Topic, UserRegistration, Role, Post
@@ -212,29 +213,13 @@ def admin_communities():
                            site=g.site)
 
 
-def topics_for_form():
-    topics = Topic.query.order_by(Topic.name).all()
-    result = [(0, _('None'))]
-    for topic in topics:
-        result.append((topic.id, topic.name))
-    return result
-
-
-def communities_for_form():
-    communities = Community.query.order_by(Community.title).all()
-    result = [(0, _('None'))]
-    for community in communities:
-        result.append((community.id, community.title))
-    return result
-
-
 @bp.route('/community/<int:community_id>/edit', methods=['GET', 'POST'])
 @login_required
 @permission_required('administer all communities')
 def admin_community_edit(community_id):
     form = EditCommunityForm()
     community = Community.query.get_or_404(community_id)
-    form.topic.choices = topics_for_form()
+    form.topic.choices = topics_for_form(0)
     if form.validate_on_submit():
         community.name = form.url.data
         community.title = form.title.data
@@ -253,6 +238,7 @@ def admin_community_edit(community_id):
         community.content_retention = form.content_retention.data
         community.topic_id = form.topic.data if form.topic.data != 0 else None
         community.default_layout = form.default_layout.data
+
         icon_file = request.files['icon_file']
         if icon_file and icon_file.filename != '':
             if community.icon_id:
@@ -268,6 +254,8 @@ def admin_community_edit(community_id):
             if file:
                 community.image = file
 
+        db.session.commit()
+        community.topic.num_communities = community.topic.communities.count()
         db.session.commit()
         flash(_('Saved'))
         return redirect(url_for('admin.admin_communities'))
@@ -341,7 +329,7 @@ def unsubscribe_everyone_then_delete_task(community_id):
 @login_required
 @permission_required('administer all communities')
 def admin_topics():
-    topics = Topic.query.order_by(Topic.name).all()
+    topics = topic_tree()
     return render_template('admin/topics.html', title=_('Topics'), topics=topics,
                            moderating_communities=moderating_communities(current_user.get_id()),
                            joined_communities=joined_communities(current_user.get_id()),
@@ -354,16 +342,16 @@ def admin_topics():
 @permission_required('administer all communities')
 def admin_topic_add():
     form = EditTopicForm()
-    form.add_community.choices = communities_for_form()
+    form.parent_id.choices = topics_for_form(0)
     if form.validate_on_submit():
         topic = Topic(name=form.name.data, machine_name=form.machine_name.data, num_communities=0)
+        if form.parent_id.data:
+            topic.parent_id = form.parent_id.data
+        else:
+            topic.parent_id = None
         db.session.add(topic)
         db.session.commit()
-        if form.add_community.data:
-            community = Community.query.get(form.add_community.data)
-            community.topic_id = topic.id
-            topic.num_communities += 1
-            db.session.commit()
+
         flash(_('Saved'))
         return redirect(url_for('admin.admin_topics'))
 
@@ -379,20 +367,22 @@ def admin_topic_add():
 def admin_topic_edit(topic_id):
     form = EditTopicForm()
     topic = Topic.query.get_or_404(topic_id)
-    form.add_community.choices = communities_for_form()
+    form.parent_id.choices = topics_for_form(topic_id)
     if form.validate_on_submit():
         topic.name = form.name.data
-        topic.num_communities = topic.communities.count() + 1
+        topic.num_communities = topic.communities.count()
         topic.machine_name = form.machine_name.data
-        if form.add_community.data:
-            community = Community.query.get(form.add_community.data)
-            community.topic_id = topic.id
+        if form.parent_id.data:
+            topic.parent_id = form.parent_id.data
+        else:
+            topic.parent_id = None
         db.session.commit()
         flash(_('Saved'))
         return redirect(url_for('admin.admin_topics'))
     else:
         form.name.data = topic.name
         form.machine_name.data = topic.machine_name
+        form.parent_id.data = topic.parent_id
     return render_template('admin/edit_topic.html', title=_('Edit topic'), form=form, topic=topic,
                            moderating_communities=moderating_communities(current_user.get_id()),
                            joined_communities=joined_communities(current_user.get_id()),
@@ -575,9 +565,7 @@ def admin_user_edit(user_id):
         user.ap_manually_approves_followers = form.manually_approves_followers.data
 
         # Update user roles. The UI only lets the user choose 1 role but the DB structure allows for multiple roles per user.
-        for role in user.roles:
-            if role.id != form.role.data:
-                user.roles.remove(role)
+        db.session.execute(text('DELETE FROM user_role WHERE user_id = :user_id'), {'user_id': user.id})
         user.roles.append(Role.query.get(form.role.data))
         if form.role.data == 4:
             flash(_("Permissions are cached for 50 seconds so new admin roles won't take effect immediately."))
@@ -585,7 +573,7 @@ def admin_user_edit(user_id):
         db.session.commit()
         user.flush_cache()
         flash(_('Saved'))
-        return redirect(url_for('admin.admin_users'))
+        return redirect(url_for('admin.admin_users', local_remote='local' if user.is_local() else 'remote'))
     else:
         if not user.is_local():
             flash(_('This is a remote user - most settings here will be regularly overwritten with data from the original server.'), 'warning')
@@ -602,7 +590,7 @@ def admin_user_edit(user_id):
         form.searchable.data = user.searchable
         form.indexable.data = user.indexable
         form.manually_approves_followers.data = user.ap_manually_approves_followers
-        if user.roles:
+        if user.roles and user.roles.count() > 0:
             form.role.data = user.roles[0].id
 
     return render_template('admin/edit_user.html', title=_('Edit user'), form=form, user=user,

app/admin/util.py

@@ -1,11 +1,14 @@
+from typing import List, Tuple
+
 from flask import request, abort, g, current_app, json, flash, render_template
 from flask_login import current_user
 from sqlalchemy import text, desc
+from flask_babel import _
 
 from app import db, cache, celery
 from app.activitypub.signature import post_request
 from app.activitypub.util import default_context
-from app.models import User, Community, Instance, Site, ActivityPubLog, CommunityMember
+from app.models import User, Community, Instance, Site, ActivityPubLog, CommunityMember, Topic
 from app.utils import gibberish
 
 
@@ -101,3 +104,39 @@ def send_newsletter(form):
 
         if form.test.data:
             break
+
+
+# replies to a post, in a tree, sorted by a variety of methods
+def topic_tree() -> List:
+    topics = Topic.query.order_by(Topic.name)
+
+    topics_dict = {topic.id: {'topic': topic, 'children': []} for topic in topics.all()}
+
+    for topic in topics:
+        if topic.parent_id is not None:
+            parent_comment = topics_dict.get(topic.parent_id)
+            if parent_comment:
+                parent_comment['children'].append(topics_dict[topic.id])
+
+    return [topic for topic in topics_dict.values() if topic['topic'].parent_id is None]
+
+
+def topics_for_form(current_topic: int) -> List[Tuple[int, str]]:
+    result = [(0, _('None'))]
+    topics = topic_tree()
+    for topic in topics:
+        if topic['topic'].id != current_topic:
+            result.append((topic['topic'].id, topic['topic'].name))
+        if topic['children']:
+            result.extend(topics_for_form_children(topic['children'], current_topic, 1))
+    return result
+
+
+def topics_for_form_children(topics, current_topic: int, depth: int) -> List[Tuple[int, str]]:
+    result = []
+    for topic in topics:
+        if topic['topic'].id != current_topic:
+            result.append((topic['topic'].id, '--' * depth + ' ' + topic['topic'].name))
+        if topic['children']:
+            result.extend(topics_for_form_children(topic['children'], current_topic, depth + 1))
+    return result

app/cli.py

@@ -144,6 +144,9 @@ def register(app):
             user_name = input("Admin user name (ideally not 'admin'): ")
             email = input("Admin email address: ")
             password = input("Admin password: ")
+            while '@' in user_name:
+                print('User name cannot be an email address.')
+                user_name = input("Admin user name (ideally not 'admin'): ")
             verification_token = random_token(16)
             private_key, public_key = RsaKeys.generate_keypair()
             admin_user = User(user_name=user_name, title=user_name,
@@ -304,6 +307,12 @@ def register(app):
                             account.bounces += 1
                     db.session.commit()
 
+    @app.cli.command("clean_up_old_activities")
+    def clean_up_old_activities():
+        with app.app_context():
+            db.session.query(ActivityPubLog).filter(ActivityPubLog.created_at < utcnow() - timedelta(days=3)).delete()
+            db.session.commit()
+
 
 def parse_communities(interests_source, segment):
     lines = interests_source.split("\n")

app/main/routes.py

@@ -189,7 +189,7 @@ def list_local_communities():
     communities = Community.query.filter_by(ap_id=None, banned=False)
     return render_template('list_communities.html', communities=communities.order_by(sort_by).all(), title=_('Local communities'), sort_by=sort_by,
                            SUBSCRIPTION_PENDING=SUBSCRIPTION_PENDING, SUBSCRIPTION_MEMBER=SUBSCRIPTION_MEMBER,
-                            SUBSCRIPTION_MODERATOR=SUBSCRIPTION_MODERATOR,
+                           SUBSCRIPTION_OWNER=SUBSCRIPTION_OWNER, SUBSCRIPTION_MODERATOR=SUBSCRIPTION_MODERATOR,
                            low_bandwidth=request.cookies.get('low_bandwidth', '0') == '1', moderating_communities=moderating_communities(current_user.get_id()),
                            joined_communities=joined_communities(current_user.get_id()))
 
@@ -204,7 +204,7 @@ def list_subscribed_communities():
         communities = []
     return render_template('list_communities.html', communities=communities, title=_('Joined communities'),
                            SUBSCRIPTION_PENDING=SUBSCRIPTION_PENDING, SUBSCRIPTION_MEMBER=SUBSCRIPTION_MEMBER, sort_by=sort_by,
-                           SUBSCRIPTION_MODERATOR=SUBSCRIPTION_MODERATOR,
+                           SUBSCRIPTION_OWNER=SUBSCRIPTION_OWNER, SUBSCRIPTION_MODERATOR=SUBSCRIPTION_MODERATOR,
                            low_bandwidth=request.cookies.get('low_bandwidth', '0') == '1', moderating_communities=moderating_communities(current_user.get_id()),
                            joined_communities=joined_communities(current_user.get_id()))
 

app/models.py

@@ -172,7 +172,7 @@ class File(db.Model):
     file_name = db.Column(db.String(255))
     width = db.Column(db.Integer)
     height = db.Column(db.Integer)
-    alt_text = db.Column(db.String(256))
+    alt_text = db.Column(db.String(300))
     source_url = db.Column(db.String(1024))
     thumbnail_path = db.Column(db.String(255))
     thumbnail_width = db.Column(db.Integer)
@@ -338,7 +338,7 @@ class Community(db.Model):
         if self.ap_id is None:
             return self.name
         else:
-            return self.ap_id
+            return self.ap_id.lower()
 
     @cache.memoize(timeout=30)
     def moderators(self):
@@ -376,7 +376,8 @@ class Community(db.Model):
 
 
     def profile_id(self):
-        return self.ap_profile_id if self.ap_profile_id else f"https://{current_app.config['SERVER_NAME']}/c/{self.name}"
+        retval = self.ap_profile_id if self.ap_profile_id else f"https://{current_app.config['SERVER_NAME']}/c/{self.name}"
+        return retval.lower()
 
     def is_local(self):
         return self.ap_id is None or self.profile_id().startswith('https://' + current_app.config['SERVER_NAME'])
@@ -678,7 +679,8 @@ class User(UserMixin, db.Model):
             join(CommunityMember).filter(CommunityMember.is_banned == False, CommunityMember.user_id == self.id).all()
 
     def profile_id(self):
-        return self.ap_profile_id if self.ap_profile_id else f"https://{current_app.config['SERVER_NAME']}/u/{self.user_name}"
+        result = self.ap_profile_id if self.ap_profile_id else f"https://{current_app.config['SERVER_NAME']}/u/{self.user_name}"
+        return result
 
     def created_recently(self):
         return self.created and self.created > utcnow() - timedelta(days=7)

app/templates/admin/topics.html

@@ -11,6 +11,24 @@
         {% include 'admin/_nav.html' %}
     </div>
 </div>
+{% macro render_topic(topic, depth) %}
+    <tr>
+        <td nowrap="nowrap">{{ '--' * depth }} {{ topic['topic'].name }}</td>
+        <td>{{ topic['topic'].num_communities  }}</td>
+        <td><a href="{{ url_for('admin.admin_topic_edit', topic_id=topic['topic'].id) }}">Edit</a> |
+            {% if topic['topic'].num_communities == 0 %}
+                <a href="{{ url_for('admin.admin_topic_delete', topic_id=topic['topic'].id) }}" class="confirm_first">Delete</a>
+            {% else %}
+                Delete
+            {% endif %}
+        </td>
+    </tr>
+    {% if topic['children'] %}
+        {% for topic in topic['children'] %}
+            {{ render_topic(topic, depth + 1)|safe }}
+        {% endfor %}
+    {% endif %}
+{% endmacro %}
 
 <div class="row">
     <div class="col">
@@ -22,13 +40,7 @@
                <th>Actions</th>
             </tr>
             {% for topic in topics %}
-                <tr>
-                    <td>{{ topic.name }}</td>
-                    <td>{{ topic.num_communities  }}</td>
-                    <td><a href="{{ url_for('admin.admin_topic_edit', topic_id=topic.id) }}">Edit</a> |
-                        <a href="{{ url_for('admin.admin_topic_delete', topic_id=topic.id) }}" class="confirm_first">Delete</a>
-                    </td>
-                </tr>
+                {{ render_topic(topic, 0)|safe }}
             {% endfor %}
         </table>
     </div>

email_notifs.sh

@@ -4,3 +4,4 @@ source venv/bin/activate
 export FLASK_APP=pyfedi.py
 flask send_missed_notifs
 flask process_email_bounces
+flask clean_up_old_activities

migrations/versions/e72aa356e4d0_increase_alt_text_length.py

@@ -0,0 +1,38 @@
+"""increase alt text length
+
+Revision ID: e72aa356e4d0
+Revises: a88efa63415b
+Create Date: 2024-03-04 12:12:07.458127
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'e72aa356e4d0'
+down_revision = 'a88efa63415b'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('file', schema=None) as batch_op:
+        batch_op.alter_column('alt_text',
+               existing_type=sa.VARCHAR(length=256),
+               type_=sa.String(length=300),
+               existing_nullable=True)
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('file', schema=None) as batch_op:
+        batch_op.alter_column('alt_text',
+               existing_type=sa.String(length=300),
+               type_=sa.VARCHAR(length=256),
+               existing_nullable=True)
+
+    # ### end Alembic commands ###