diff --git a/app/activitypub/util.py b/app/activitypub/util.py index 735c1199..1af84b90 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -492,23 +492,24 @@ def post_json_to_model(post_json, user, community) -> Post: domain = domain_from_url(post.url) # notify about links to banned websites. already_notified = set() # often admins and mods are the same people - avoid notifying them twice - if domain.notify_mods: - for community_member in post.community.moderators(): - notify = Notification(title='Suspicious content', url=post.ap_id, user_id=community_member.user_id, author_id=user.id) - db.session.add(notify) - already_notified.add(community_member.user_id) - - if domain.notify_admins: - for admin in Site.admins(): - if admin.id not in already_notified: - notify = Notification(title='Suspicious content', url=post.ap_id, user_id=admin.id, author_id=user.id) + if domain: + if domain.notify_mods: + for community_member in post.community.moderators(): + notify = Notification(title='Suspicious content', url=post.ap_id, user_id=community_member.user_id, author_id=user.id) db.session.add(notify) - admin.unread_notifications += 1 - if domain.banned: - post = None - if not domain.banned: - domain.post_count += 1 - post.domain = domain + already_notified.add(community_member.user_id) + + if domain.notify_admins: + for admin in Site.admins(): + if admin.id not in already_notified: + notify = Notification(title='Suspicious content', url=post.ap_id, user_id=admin.id, author_id=user.id) + db.session.add(notify) + admin.unread_notifications += 1 + if domain.banned: + post = None + if not domain.banned: + domain.post_count += 1 + post.domain = domain if 'image' in post_json and post: image = File(source_url=post_json['image']['url']) db.session.add(image) @@ -694,7 +695,7 @@ def find_instance_id(server): else: # Our instance does not know about {server} yet. Initially, create a sparse row in the 'instance' table and spawn a background # task to update the row with more details later - new_instance = Instance(domain=server, software='unknown', created_at=utcnow()) + new_instance = Instance(domain=server, software='unknown', created_at=utcnow(), trusted=server == 'piefed.social') db.session.add(new_instance) db.session.commit() diff --git a/app/auth/routes.py b/app/auth/routes.py index 7fb5e87d..04fea620 100644 --- a/app/auth/routes.py +++ b/app/auth/routes.py @@ -106,7 +106,8 @@ def register(): flash(_('Your username contained special letters so it was changed to %(name)s.', name=form.user_name.data), 'warning') user = User(user_name=form.user_name.data, title=form.user_name.data, email=form.real_email.data, verification_token=verification_token, instance_id=1, ip_address=ip_address(), - banned=user_ip_banned() or user_cookie_banned()) + banned=user_ip_banned() or user_cookie_banned(), email_unread_sent=False, email_messages_sent=False, + referrer=session.get('Referer')) user.set_password(form.password.data) db.session.add(user) db.session.commit() diff --git a/app/cli.py b/app/cli.py index a8b29d7b..fef26de8 100644 --- a/app/cli.py +++ b/app/cli.py @@ -2,7 +2,10 @@ # e.g. export FLASK_APP=pyfedi.py from datetime import datetime, timedelta -from flask import json +import flask +from flask import json, current_app +from flask_babel import _ +from sqlalchemy import or_, desc from app import db import click @@ -10,10 +13,10 @@ import os from app.activitypub.signature import RsaKeys from app.auth.util import random_token -from app.email import send_verification_email +from app.email import send_verification_email, send_email from app.models import Settings, BannedInstances, Interest, Role, User, RolePermission, Domain, ActivityPubLog, \ - utcnow, Site, Instance, File -from app.utils import file_get_contents, retrieve_block_list + utcnow, Site, Instance, File, Notification, Post, CommunityMember +from app.utils import file_get_contents, retrieve_block_list, blocked_domains def register(app): @@ -167,6 +170,51 @@ def register(app): if f is None: os.unlink(file_path) + @app.cli.command("send_missed_notifs") + def send_missed_notifs(): + with app.app_context(): + users_to_notify = User.query.join(Notification, User.id == Notification.user_id).filter( + User.ap_id == None, + Notification.created_at > User.last_seen, + Notification.read == False, + User.email_unread_sent == False, # they have not been emailed since last activity + User.email_unread == True # they want to be emailed + ).all() + + for user in users_to_notify: + notifications = Notification.query.filter(Notification.user_id == user.id, Notification.read == False, + Notification.created_at > user.last_seen).all() + if notifications: + # Also get the top 20 posts since their last login + posts = Post.query.join(CommunityMember, Post.community_id == CommunityMember.community_id).filter( + CommunityMember.is_banned == False) + posts = posts.filter(CommunityMember.user_id == user.id) + if user.ignore_bots: + posts = posts.filter(Post.from_bot == False) + if user.show_nsfl is False: + posts = posts.filter(Post.nsfl == False) + if user.show_nsfw is False: + posts = posts.filter(Post.nsfw == False) + domains_ids = blocked_domains(user.id) + if domains_ids: + posts = posts.filter(or_(Post.domain_id.not_in(domains_ids), Post.domain_id == None)) + posts = posts.filter(Post.posted_at > user.last_seen).order_by(desc(Post.score)) + posts = posts.limit(20).all() + + # Send email! + send_email(_('You have unread notifications'), + sender='PieFed ', + recipients=[user.email], + text_body=flask.render_template('email/unread_notifications.txt', user=user, + notifications=notifications), + html_body=flask.render_template('email/unread_notifications.html', user=user, + notifications=notifications, + posts=posts, + domain=current_app.config['SERVER_NAME'])) + user.email_unread_sent = True + db.session.commit() + + def parse_communities(interests_source, segment): lines = interests_source.split("\n") include_in_output = False diff --git a/app/main/routes.py b/app/main/routes.py index fca8e7d0..f565b7be 100644 --- a/app/main/routes.py +++ b/app/main/routes.py @@ -3,12 +3,14 @@ from datetime import datetime, timedelta from math import log from random import randint -from sqlalchemy.sql.operators import or_ +import flask +from sqlalchemy.sql.operators import or_, and_ from app import db, cache from app.activitypub.util import default_context, make_image_sizes_async, refresh_user_profile, find_actor_or_create from app.constants import SUBSCRIPTION_PENDING, SUBSCRIPTION_MEMBER, POST_TYPE_IMAGE, POST_TYPE_LINK, \ SUBSCRIPTION_OWNER, SUBSCRIPTION_MODERATOR +from app.email import send_email from app.inoculation import inoculation from app.main import bp from flask import g, session, flash, request, current_app, url_for, redirect, make_response, jsonify @@ -257,7 +259,48 @@ def list_files(directory): @bp.route('/test') def test(): - u = User.query.filter(User.email_unread == True).join(Notification, Notification.user_id == User.id).filter() + return '' + users_to_notify = User.query.join(Notification, User.id == Notification.user_id).filter( + User.ap_id == None, + Notification.created_at > User.last_seen, + Notification.read == False, + User.email_unread_sent == False, # they have not been emailed since last activity + User.email_unread == True # they want to be emailed + ).all() + + for user in users_to_notify: + notifications = Notification.query.filter(Notification.user_id == user.id, Notification.read == False, + Notification.created_at > user.last_seen).all() + if notifications: + # Also get the top 20 posts since their last login + posts = Post.query.join(CommunityMember, Post.community_id == CommunityMember.community_id).filter( + CommunityMember.is_banned == False) + posts = posts.filter(CommunityMember.user_id == user.id) + if user.ignore_bots: + posts = posts.filter(Post.from_bot == False) + if user.show_nsfl is False: + posts = posts.filter(Post.nsfl == False) + if user.show_nsfw is False: + posts = posts.filter(Post.nsfw == False) + domains_ids = blocked_domains(user.id) + if domains_ids: + posts = posts.filter(or_(Post.domain_id.not_in(domains_ids), Post.domain_id == None)) + posts = posts.filter(Post.posted_at > user.last_seen).order_by(desc(Post.score)) + posts = posts.limit(20).all() + + # Send email! + send_email(_('You have unread notifications'), + sender='PieFed ', + recipients=[user.email], + text_body=flask.render_template('email/unread_notifications.txt', user=user, notifications=notifications), + html_body=flask.render_template('email/unread_notifications.html', user=user, + notifications=notifications, + posts=posts, + domain=current_app.config['SERVER_NAME'])) + user.email_unread_sent = True + db.session.commit() + + return 'ok' diff --git a/app/models.py b/app/models.py index bc77d993..4a6ec936 100644 --- a/app/models.py +++ b/app/models.py @@ -61,6 +61,7 @@ class Instance(db.Model): start_trying_again = db.Column(db.DateTime) # When to start trying again. Should grow exponentially with each failure. gone_forever = db.Column(db.Boolean, default=False) # True once this instance is considered offline forever - never start trying again ip_address = db.Column(db.String(50)) + trusted = db.Column(db.Boolean, default=False) posts = db.relationship('Post', backref='instance', lazy='dynamic') post_replies = db.relationship('PostReply', backref='instance', lazy='dynamic') @@ -261,6 +262,7 @@ class Community(db.Model): ap_fetched_at = db.Column(db.DateTime) ap_deleted_at = db.Column(db.DateTime) ap_inbox_url = db.Column(db.String(255)) + ap_outbox_url = db.Column(db.String(255)) ap_moderators_url = db.Column(db.String(255)) ap_domain = db.Column(db.String(255)) @@ -443,6 +445,9 @@ class User(UserMixin, db.Model): public_key = db.Column(db.Text) private_key = db.Column(db.Text) newsletter = db.Column(db.Boolean, default=True) + email_unread = db.Column(db.Boolean, default=True) # True if they want to receive 'unread notifications' emails + email_unread_sent = db.Column(db.Boolean) # True after a 'unread notifications' email has been sent. None for remote users + receive_message_mode = db.Column(db.String(20), default='Closed') # possible values: Open, TrustedOnly, Closed bounces = db.Column(db.SmallInteger, default=0) timezone = db.Column(db.String(20)) reputation = db.Column(db.Float, default=0.0) @@ -459,6 +464,7 @@ class User(UserMixin, db.Model): reports = db.Column(db.Integer, default=0) # how many times this user has been reported. default_sort = db.Column(db.String(25), default='hot') theme = db.Column(db.String(20), default='') + referrer = db.Column(db.String(256)) avatar = db.relationship('File', lazy='joined', foreign_keys=[avatar_id], single_parent=True, cascade="all, delete-orphan") cover = db.relationship('File', lazy='joined', foreign_keys=[cover_id], single_parent=True, cascade="all, delete-orphan") diff --git a/app/templates/admin/approve_registrations.html b/app/templates/admin/approve_registrations.html index 454b99f6..a22d3bda 100644 --- a/app/templates/admin/approve_registrations.html +++ b/app/templates/admin/approve_registrations.html @@ -27,6 +27,7 @@ Answer Applied IP + Source Actions {% for registration in registrations %} @@ -38,6 +39,7 @@ {{ registration.answer }} {{ moment(registration.created_at).fromNow() }} {{ registration.user.ip_address if registration.user.ip_address }} + {{ registration.user.referrer if registration.user.referrer }} {{ _('Approve') }} {{ _('View') }} | {{ _('Delete') }} diff --git a/app/templates/admin/users.html b/app/templates/admin/users.html index c73eee35..2fc23539 100644 --- a/app/templates/admin/users.html +++ b/app/templates/admin/users.html @@ -30,6 +30,7 @@ Banned Reports IP + Source Actions {% for user in users.items %} @@ -48,6 +49,7 @@ {{ 'Banned'|safe if user.banned }} {{ user.reports if user.reports > 0 }} {{ user.ip_address if user.ip_address }} + {{ user.referrer if user.referrer }} View local | {% if not user.is_local() %} View remote | diff --git a/app/templates/email/unread_notifications.html b/app/templates/email/unread_notifications.html new file mode 100644 index 00000000..ac2c13ba --- /dev/null +++ b/app/templates/email/unread_notifications.html @@ -0,0 +1,19 @@ +

+

Hi {{ user.display_name() }},

+

Here's some notifications you've missed since your last visit:

+ +

Mark all as read

+{% if posts %} +

Also here's a few recent posts:

+ +{% endif %} +

Unsubscribe from these emails by un-ticking the 'Receive email + about missed notifications' checkbox.

\ No newline at end of file diff --git a/app/templates/email/unread_notifications.txt b/app/templates/email/unread_notifications.txt new file mode 100644 index 00000000..f8e06ba4 --- /dev/null +++ b/app/templates/email/unread_notifications.txt @@ -0,0 +1,8 @@ +Hi {{ user.display_name() }}, + +Here's some notifications you've missed since your last visit: + {% for notification in notifications %} + - {{ notification.title }} - {{ url_for('user.notification_goto', notification_id=notification.id, _external=True) }} + {% endfor %} + +Unsubscribe from these emails by un-ticking the 'Receive email about missed notifications' checkbox at {{ url_for('user.change_settings', _external=True) }}. diff --git a/app/templates/user/edit_settings.html b/app/templates/user/edit_settings.html index 0019ff5e..6f002bf3 100644 --- a/app/templates/user/edit_settings.html +++ b/app/templates/user/edit_settings.html @@ -20,6 +20,7 @@
{{ form.csrf_token() }} {{ render_field(form.newsletter) }} + {{ render_field(form.email_unread) }} {{ render_field(form.ignore_bots) }} {{ render_field(form.nsfw) }} {{ render_field(form.nsfl) }} diff --git a/app/user/forms.py b/app/user/forms.py index beac7266..c1d0e17a 100644 --- a/app/user/forms.py +++ b/app/user/forms.py @@ -32,6 +32,7 @@ class ProfileForm(FlaskForm): class SettingsForm(FlaskForm): newsletter = BooleanField(_l('Subscribe to email newsletter')) + email_unread = BooleanField(_l('Receive email about missed notifications')) ignore_bots = BooleanField(_l('Hide posts by bots')) nsfw = BooleanField(_l('Show NSFW posts')) nsfl = BooleanField(_l('Show NSFL posts')) diff --git a/app/user/routes.py b/app/user/routes.py index 7f1ec83f..6d378c90 100644 --- a/app/user/routes.py +++ b/app/user/routes.py @@ -162,6 +162,7 @@ def change_settings(): current_user.indexable = form.indexable.data current_user.default_sort = form.default_sort.data current_user.theme = form.theme.data + current_user.email_unread = form.email_unread.data import_file = request.files['import_file'] if import_file and import_file.filename != '': file_ext = os.path.splitext(import_file.filename)[1] @@ -186,6 +187,7 @@ def change_settings(): return redirect(url_for('user.change_settings')) elif request.method == 'GET': form.newsletter.data = current_user.newsletter + form.email_unread.data = current_user.email_unread form.ignore_bots.data = current_user.ignore_bots form.nsfw.data = current_user.show_nsfw form.nsfl.data = current_user.show_nsfl diff --git a/app/utils.py b/app/utils.py index 9f60e946..cb870356 100644 --- a/app/utils.py +++ b/app/utils.py @@ -254,12 +254,15 @@ def markdown_to_text(markdown_text) -> str: def domain_from_url(url: str, create=True) -> Domain: parsed_url = urlparse(url.lower().replace('www.', '')) - domain = Domain.query.filter_by(name=parsed_url.hostname.lower()).first() - if create and domain is None: - domain = Domain(name=parsed_url.hostname.lower()) - db.session.add(domain) - db.session.commit() - return domain + if parsed_url and parsed_url.hostname: + domain = Domain.query.filter_by(name=parsed_url.hostname.lower()).first() + if create and domain is None: + domain = Domain(name=parsed_url.hostname.lower()) + db.session.add(domain) + db.session.commit() + return domain + else: + return None def shorten_string(input_str, max_length=50): diff --git a/migrations/versions/1505d32771b7_trusted_instances.py b/migrations/versions/1505d32771b7_trusted_instances.py new file mode 100644 index 00000000..7b7fc7de --- /dev/null +++ b/migrations/versions/1505d32771b7_trusted_instances.py @@ -0,0 +1,44 @@ +"""trusted instances + +Revision ID: 1505d32771b7 +Revises: a937c8721612 +Create Date: 2024-02-23 16:26:02.561074 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '1505d32771b7' +down_revision = 'a937c8721612' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('instance', schema=None) as batch_op: + batch_op.add_column(sa.Column('trusted', sa.Boolean(), nullable=True)) + + with op.batch_alter_table('user', schema=None) as batch_op: + batch_op.add_column(sa.Column('receive_message_mode', sa.String(length=20), nullable=True)) + batch_op.add_column(sa.Column('referrer', sa.String(length=256), nullable=True)) + batch_op.drop_column('email_messages_sent') + batch_op.drop_column('email_messages') + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('user', schema=None) as batch_op: + batch_op.add_column(sa.Column('email_messages', sa.BOOLEAN(), autoincrement=False, nullable=True)) + batch_op.add_column(sa.Column('email_messages_sent', sa.BOOLEAN(), autoincrement=False, nullable=True)) + batch_op.drop_column('referrer') + batch_op.drop_column('receive_message_mode') + + with op.batch_alter_table('instance', schema=None) as batch_op: + batch_op.drop_column('trusted') + + # ### end Alembic commands ### diff --git a/pyfedi.py b/pyfedi.py index 1d306a37..a44d2c56 100644 --- a/pyfedi.py +++ b/pyfedi.py @@ -7,7 +7,7 @@ from flask_login import current_user from app import create_app, db, cli import os, click -from flask import session, g, json, request +from flask import session, g, json, request, current_app from app.constants import POST_TYPE_LINK, POST_TYPE_IMAGE, POST_TYPE_ARTICLE from app.models import Site from app.utils import getmtime, gibberish, shorten_string, shorten_url, digits, user_access, community_membership, \ @@ -55,6 +55,12 @@ def before_request(): g.site = Site.query.get(1) if current_user.is_authenticated: current_user.last_seen = datetime.utcnow() + current_user.email_unread_sent = False + else: + if session.get('Referer') is None and \ + request.headers.get('Referer') is not None and \ + current_app.config['SERVER_NAME'] not in request.headers.get('Referer'): + session['Referer'] = request.headers.get('Referer') @app.after_request