diff --git a/INSTALL.md b/INSTALL.md index 1cb32063..c90889a9 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -387,3 +387,31 @@ Once a week or so it's good to run remove_orphan_files.sh to save disk space: 5 4 * * 1 rimu cd /home/rimu/pyfedi && /home/rimu/pyfedi/remove_orphan_files.sh ``` +--- + + +Email +--- + +Email can be sent either through SMTP or Amazon web services (SES). SES is faster but PieFed does not send much +email so it probably doesn't matter which method you choose. + +### AWS SES + +PieFed uses Amazon's "boto3" module to connect to SES. Boto3 needs to log into AWS and that can be set up using a file +at ~/.aws/credentials or environment variables. Details at https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html. + +In your .env you need to set the AWS region you're using for SES. Something like AWS_REGION = 'ap-southeast-2'. + +### SMTP + +To use SMTP you need to set all the MAIL_* environment variables in you .env file. See env.sample for a list of them. + +### Testing email + +You need to set MAIL_FROM in .env to some email address. + +Log into Piefed then go to https://yourdomain/test_email to trigger a test email. It will use SES or SMTP depending on +which environment variables you defined in .env. If MAIL_SERVER is empty it will try SES. Then if AWS_REGION is empty it'll +silently do nothing. + diff --git a/app/activitypub/util.py b/app/activitypub/util.py index 2ff418e1..67be217a 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -1222,6 +1222,9 @@ def create_post(activity_log: ActivityPubLog, community: Community, request_json post.url = request_json['object']['attachment'][0]['href'] if is_image_url(post.url): post.type = POST_TYPE_IMAGE + image = File(source_url=request_json['object']['image']['url']) + db.session.add(image) + post.image = image else: post.type = POST_TYPE_LINK domain = domain_from_url(post.url) @@ -1247,7 +1250,7 @@ def create_post(activity_log: ActivityPubLog, community: Community, request_json if not domain.banned: domain.post_count += 1 post.domain = domain - if 'image' in request_json['object']: + if 'image' in request_json['object'] and post.image is None: image = File(source_url=request_json['object']['image']['url']) db.session.add(image) post.image = image diff --git a/app/admin/forms.py b/app/admin/forms.py index 6a759280..1a50d314 100644 --- a/app/admin/forms.py +++ b/app/admin/forms.py @@ -95,7 +95,7 @@ class EditCommunityForm(FlaskForm): class EditTopicForm(FlaskForm): name = StringField(_l('Name'), validators=[DataRequired()]) machine_name = StringField(_l('Url'), validators=[DataRequired()]) - add_community = SelectField(_l('Community to add'), coerce=int, validators=[Optional()]) + parent_id = SelectField(_l('Parent topic'), coerce=int, validators=[Optional()]) submit = SubmitField(_l('Save')) diff --git a/app/admin/routes.py b/app/admin/routes.py index b09292e9..1c4ac642 100644 --- a/app/admin/routes.py +++ b/app/admin/routes.py @@ -12,7 +12,8 @@ from app.activitypub.signature import post_request from app.activitypub.util import default_context from app.admin.forms import FederationForm, SiteMiscForm, SiteProfileForm, EditCommunityForm, EditUserForm, \ EditTopicForm, SendNewsletterForm, AddUserForm -from app.admin.util import unsubscribe_from_everything_then_delete, unsubscribe_from_community, send_newsletter +from app.admin.util import unsubscribe_from_everything_then_delete, unsubscribe_from_community, send_newsletter, \ + topic_tree, topics_for_form from app.community.util import save_icon_file, save_banner_file from app.models import AllowedInstances, BannedInstances, ActivityPubLog, utcnow, Site, Community, CommunityMember, \ User, Instance, File, Report, Topic, UserRegistration, Role, Post @@ -212,29 +213,13 @@ def admin_communities(): site=g.site) -def topics_for_form(): - topics = Topic.query.order_by(Topic.name).all() - result = [(0, _('None'))] - for topic in topics: - result.append((topic.id, topic.name)) - return result - - -def communities_for_form(): - communities = Community.query.order_by(Community.title).all() - result = [(0, _('None'))] - for community in communities: - result.append((community.id, community.title)) - return result - - @bp.route('/community//edit', methods=['GET', 'POST']) @login_required @permission_required('administer all communities') def admin_community_edit(community_id): form = EditCommunityForm() community = Community.query.get_or_404(community_id) - form.topic.choices = topics_for_form() + form.topic.choices = topics_for_form(0) if form.validate_on_submit(): community.name = form.url.data community.title = form.title.data @@ -253,6 +238,7 @@ def admin_community_edit(community_id): community.content_retention = form.content_retention.data community.topic_id = form.topic.data if form.topic.data != 0 else None community.default_layout = form.default_layout.data + icon_file = request.files['icon_file'] if icon_file and icon_file.filename != '': if community.icon_id: @@ -268,6 +254,8 @@ def admin_community_edit(community_id): if file: community.image = file + db.session.commit() + community.topic.num_communities = community.topic.communities.count() db.session.commit() flash(_('Saved')) return redirect(url_for('admin.admin_communities')) @@ -341,7 +329,7 @@ def unsubscribe_everyone_then_delete_task(community_id): @login_required @permission_required('administer all communities') def admin_topics(): - topics = Topic.query.order_by(Topic.name).all() + topics = topic_tree() return render_template('admin/topics.html', title=_('Topics'), topics=topics, moderating_communities=moderating_communities(current_user.get_id()), joined_communities=joined_communities(current_user.get_id()), @@ -354,16 +342,16 @@ def admin_topics(): @permission_required('administer all communities') def admin_topic_add(): form = EditTopicForm() - form.add_community.choices = communities_for_form() + form.parent_id.choices = topics_for_form(0) if form.validate_on_submit(): topic = Topic(name=form.name.data, machine_name=form.machine_name.data, num_communities=0) + if form.parent_id.data: + topic.parent_id = form.parent_id.data + else: + topic.parent_id = None db.session.add(topic) db.session.commit() - if form.add_community.data: - community = Community.query.get(form.add_community.data) - community.topic_id = topic.id - topic.num_communities += 1 - db.session.commit() + flash(_('Saved')) return redirect(url_for('admin.admin_topics')) @@ -379,20 +367,22 @@ def admin_topic_add(): def admin_topic_edit(topic_id): form = EditTopicForm() topic = Topic.query.get_or_404(topic_id) - form.add_community.choices = communities_for_form() + form.parent_id.choices = topics_for_form(topic_id) if form.validate_on_submit(): topic.name = form.name.data - topic.num_communities = topic.communities.count() + 1 + topic.num_communities = topic.communities.count() topic.machine_name = form.machine_name.data - if form.add_community.data: - community = Community.query.get(form.add_community.data) - community.topic_id = topic.id + if form.parent_id.data: + topic.parent_id = form.parent_id.data + else: + topic.parent_id = None db.session.commit() flash(_('Saved')) return redirect(url_for('admin.admin_topics')) else: form.name.data = topic.name form.machine_name.data = topic.machine_name + form.parent_id.data = topic.parent_id return render_template('admin/edit_topic.html', title=_('Edit topic'), form=form, topic=topic, moderating_communities=moderating_communities(current_user.get_id()), joined_communities=joined_communities(current_user.get_id()), @@ -575,9 +565,7 @@ def admin_user_edit(user_id): user.ap_manually_approves_followers = form.manually_approves_followers.data # Update user roles. The UI only lets the user choose 1 role but the DB structure allows for multiple roles per user. - for role in user.roles: - if role.id != form.role.data: - user.roles.remove(role) + db.session.execute(text('DELETE FROM user_role WHERE user_id = :user_id'), {'user_id': user.id}) user.roles.append(Role.query.get(form.role.data)) if form.role.data == 4: flash(_("Permissions are cached for 50 seconds so new admin roles won't take effect immediately.")) @@ -585,7 +573,7 @@ def admin_user_edit(user_id): db.session.commit() user.flush_cache() flash(_('Saved')) - return redirect(url_for('admin.admin_users')) + return redirect(url_for('admin.admin_users', local_remote='local' if user.is_local() else 'remote')) else: if not user.is_local(): flash(_('This is a remote user - most settings here will be regularly overwritten with data from the original server.'), 'warning') @@ -602,7 +590,7 @@ def admin_user_edit(user_id): form.searchable.data = user.searchable form.indexable.data = user.indexable form.manually_approves_followers.data = user.ap_manually_approves_followers - if user.roles: + if user.roles and user.roles.count() > 0: form.role.data = user.roles[0].id return render_template('admin/edit_user.html', title=_('Edit user'), form=form, user=user, diff --git a/app/admin/util.py b/app/admin/util.py index c076c222..b1669ac1 100644 --- a/app/admin/util.py +++ b/app/admin/util.py @@ -1,11 +1,14 @@ +from typing import List, Tuple + from flask import request, abort, g, current_app, json, flash, render_template from flask_login import current_user from sqlalchemy import text, desc +from flask_babel import _ from app import db, cache, celery from app.activitypub.signature import post_request from app.activitypub.util import default_context -from app.models import User, Community, Instance, Site, ActivityPubLog, CommunityMember +from app.models import User, Community, Instance, Site, ActivityPubLog, CommunityMember, Topic from app.utils import gibberish @@ -101,3 +104,39 @@ def send_newsletter(form): if form.test.data: break + + +# replies to a post, in a tree, sorted by a variety of methods +def topic_tree() -> List: + topics = Topic.query.order_by(Topic.name) + + topics_dict = {topic.id: {'topic': topic, 'children': []} for topic in topics.all()} + + for topic in topics: + if topic.parent_id is not None: + parent_comment = topics_dict.get(topic.parent_id) + if parent_comment: + parent_comment['children'].append(topics_dict[topic.id]) + + return [topic for topic in topics_dict.values() if topic['topic'].parent_id is None] + + +def topics_for_form(current_topic: int) -> List[Tuple[int, str]]: + result = [(0, _('None'))] + topics = topic_tree() + for topic in topics: + if topic['topic'].id != current_topic: + result.append((topic['topic'].id, topic['topic'].name)) + if topic['children']: + result.extend(topics_for_form_children(topic['children'], current_topic, 1)) + return result + + +def topics_for_form_children(topics, current_topic: int, depth: int) -> List[Tuple[int, str]]: + result = [] + for topic in topics: + if topic['topic'].id != current_topic: + result.append((topic['topic'].id, '--' * depth + ' ' + topic['topic'].name)) + if topic['children']: + result.extend(topics_for_form_children(topic['children'], current_topic, depth + 1)) + return result diff --git a/app/cli.py b/app/cli.py index 42c5a5f6..a1f1ec19 100644 --- a/app/cli.py +++ b/app/cli.py @@ -144,6 +144,9 @@ def register(app): user_name = input("Admin user name (ideally not 'admin'): ") email = input("Admin email address: ") password = input("Admin password: ") + while '@' in user_name: + print('User name cannot be an email address.') + user_name = input("Admin user name (ideally not 'admin'): ") verification_token = random_token(16) private_key, public_key = RsaKeys.generate_keypair() admin_user = User(user_name=user_name, title=user_name, @@ -304,6 +307,12 @@ def register(app): account.bounces += 1 db.session.commit() + @app.cli.command("clean_up_old_activities") + def clean_up_old_activities(): + with app.app_context(): + db.session.query(ActivityPubLog).filter(ActivityPubLog.created_at < utcnow() - timedelta(days=3)).delete() + db.session.commit() + def parse_communities(interests_source, segment): lines = interests_source.split("\n") diff --git a/app/main/routes.py b/app/main/routes.py index 18715183..08b7cc7e 100644 --- a/app/main/routes.py +++ b/app/main/routes.py @@ -189,7 +189,7 @@ def list_local_communities(): communities = Community.query.filter_by(ap_id=None, banned=False) return render_template('list_communities.html', communities=communities.order_by(sort_by).all(), title=_('Local communities'), sort_by=sort_by, SUBSCRIPTION_PENDING=SUBSCRIPTION_PENDING, SUBSCRIPTION_MEMBER=SUBSCRIPTION_MEMBER, - SUBSCRIPTION_MODERATOR=SUBSCRIPTION_MODERATOR, + SUBSCRIPTION_OWNER=SUBSCRIPTION_OWNER, SUBSCRIPTION_MODERATOR=SUBSCRIPTION_MODERATOR, low_bandwidth=request.cookies.get('low_bandwidth', '0') == '1', moderating_communities=moderating_communities(current_user.get_id()), joined_communities=joined_communities(current_user.get_id())) @@ -204,7 +204,7 @@ def list_subscribed_communities(): communities = [] return render_template('list_communities.html', communities=communities, title=_('Joined communities'), SUBSCRIPTION_PENDING=SUBSCRIPTION_PENDING, SUBSCRIPTION_MEMBER=SUBSCRIPTION_MEMBER, sort_by=sort_by, - SUBSCRIPTION_MODERATOR=SUBSCRIPTION_MODERATOR, + SUBSCRIPTION_OWNER=SUBSCRIPTION_OWNER, SUBSCRIPTION_MODERATOR=SUBSCRIPTION_MODERATOR, low_bandwidth=request.cookies.get('low_bandwidth', '0') == '1', moderating_communities=moderating_communities(current_user.get_id()), joined_communities=joined_communities(current_user.get_id())) diff --git a/app/models.py b/app/models.py index 6b8a36ae..0bbdf1da 100644 --- a/app/models.py +++ b/app/models.py @@ -172,7 +172,7 @@ class File(db.Model): file_name = db.Column(db.String(255)) width = db.Column(db.Integer) height = db.Column(db.Integer) - alt_text = db.Column(db.String(256)) + alt_text = db.Column(db.String(300)) source_url = db.Column(db.String(1024)) thumbnail_path = db.Column(db.String(255)) thumbnail_width = db.Column(db.Integer) @@ -338,7 +338,7 @@ class Community(db.Model): if self.ap_id is None: return self.name else: - return self.ap_id + return self.ap_id.lower() @cache.memoize(timeout=30) def moderators(self): @@ -376,7 +376,8 @@ class Community(db.Model): def profile_id(self): - return self.ap_profile_id if self.ap_profile_id else f"https://{current_app.config['SERVER_NAME']}/c/{self.name}" + retval = self.ap_profile_id if self.ap_profile_id else f"https://{current_app.config['SERVER_NAME']}/c/{self.name}" + return retval.lower() def is_local(self): return self.ap_id is None or self.profile_id().startswith('https://' + current_app.config['SERVER_NAME']) @@ -678,7 +679,8 @@ class User(UserMixin, db.Model): join(CommunityMember).filter(CommunityMember.is_banned == False, CommunityMember.user_id == self.id).all() def profile_id(self): - return self.ap_profile_id if self.ap_profile_id else f"https://{current_app.config['SERVER_NAME']}/u/{self.user_name}" + result = self.ap_profile_id if self.ap_profile_id else f"https://{current_app.config['SERVER_NAME']}/u/{self.user_name}" + return result def created_recently(self): return self.created and self.created > utcnow() - timedelta(days=7) diff --git a/app/templates/admin/topics.html b/app/templates/admin/topics.html index 7bba2707..7763a83f 100644 --- a/app/templates/admin/topics.html +++ b/app/templates/admin/topics.html @@ -11,6 +11,24 @@ {% include 'admin/_nav.html' %} +{% macro render_topic(topic, depth) %} + + {{ '--' * depth }} {{ topic['topic'].name }} + {{ topic['topic'].num_communities }} + Edit | + {% if topic['topic'].num_communities == 0 %} + Delete + {% else %} + Delete + {% endif %} + + + {% if topic['children'] %} + {% for topic in topic['children'] %} + {{ render_topic(topic, depth + 1)|safe }} + {% endfor %} + {% endif %} +{% endmacro %}
@@ -22,13 +40,7 @@ Actions {% for topic in topics %} - - {{ topic.name }} - {{ topic.num_communities }} - Edit | - Delete - - + {{ render_topic(topic, 0)|safe }} {% endfor %}
diff --git a/email_notifs.sh b/email_notifs.sh index c70e21bb..9c22894f 100755 --- a/email_notifs.sh +++ b/email_notifs.sh @@ -4,3 +4,4 @@ source venv/bin/activate export FLASK_APP=pyfedi.py flask send_missed_notifs flask process_email_bounces +flask clean_up_old_activities diff --git a/migrations/versions/e72aa356e4d0_increase_alt_text_length.py b/migrations/versions/e72aa356e4d0_increase_alt_text_length.py new file mode 100644 index 00000000..bc0cac28 --- /dev/null +++ b/migrations/versions/e72aa356e4d0_increase_alt_text_length.py @@ -0,0 +1,38 @@ +"""increase alt text length + +Revision ID: e72aa356e4d0 +Revises: a88efa63415b +Create Date: 2024-03-04 12:12:07.458127 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'e72aa356e4d0' +down_revision = 'a88efa63415b' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('file', schema=None) as batch_op: + batch_op.alter_column('alt_text', + existing_type=sa.VARCHAR(length=256), + type_=sa.String(length=300), + existing_nullable=True) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('file', schema=None) as batch_op: + batch_op.alter_column('alt_text', + existing_type=sa.String(length=300), + type_=sa.VARCHAR(length=256), + existing_nullable=True) + + # ### end Alembic commands ###