From 49a3c587f90c995d12590506da8a980515300721 Mon Sep 17 00:00:00 2001 From: rra Date: Mon, 8 Apr 2024 14:22:47 +0200 Subject: [PATCH 01/24] improve about page --- app/main/routes.py | 23 ++++++++++------------- app/templates/about.html | 8 +++++--- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/app/main/routes.py b/app/main/routes.py index 9df40e6c..7d4fbac3 100644 --- a/app/main/routes.py +++ b/app/main/routes.py @@ -10,7 +10,7 @@ from sqlalchemy.sql.operators import or_, and_ from app import db, cache from app.activitypub.util import default_context, make_image_sizes_async, refresh_user_profile, find_actor_or_create, \ - refresh_community_profile_task + refresh_community_profile_task, users_total, active_month, local_posts, local_communities, local_comments from app.constants import SUBSCRIPTION_PENDING, SUBSCRIPTION_MEMBER, POST_TYPE_IMAGE, POST_TYPE_LINK, \ SUBSCRIPTION_OWNER, SUBSCRIPTION_MODERATOR from app.email import send_email, send_welcome_email @@ -48,7 +48,6 @@ def index(sort=None): def popular(sort=None): return home_page('popular', sort) - @bp.route('/all', methods=['GET']) @bp.route('/all/', methods=['GET']) def all_posts(sort=None): @@ -232,19 +231,17 @@ def donate(): @bp.route('/about') def about_page(): - users = User.query.filter_by(ap_id=None, deleted=False, banned=False).all() - user_amount = len(users) - # Todo, figure out how to filter the user list with the list of user_role user_id == 4 - #admins = users.filter() - # Todo, figure out how to filter the user list with the list of user_role user_id == 4 - #staff = users.filter() - - domains_amount = len(Domain.query.filter_by(banned=False).all()) - community_amount = len(Community.query.all()) + user_amount = users_total() + MAU = active_month() + posts_amount = local_posts() + + admins = db.session.execute(text('SELECT user_name, email FROM "user" WHERE "id" IN (SELECT "user_id" FROM "user_role" WHERE "role_id" = 4) ORDER BY id')).all() + staff = db.session.execute(text('SELECT user_name FROM "user" WHERE "id" IN (SELECT "user_id" FROM "user_role" WHERE "role_id" = 2) ORDER BY id')).all() + domains_amount = db.session.execute(text('SELECT COUNT(id) as c FROM "domain" WHERE "banned" IS false')).scalar() + community_amount = local_communities() instance = Instance.query.filter_by(id=1).first() - - return render_template('about.html', user_amount=user_amount, domains_amount=domains_amount, community_amount=community_amount, instance=instance)#, admins=admins) + return render_template('about.html', user_amount=user_amount, mau=MAU, posts_amount=posts_amount, domains_amount=domains_amount, community_amount=community_amount, instance=instance, admins=admins, staff=staff) @bp.route('/privacy') diff --git a/app/templates/about.html b/app/templates/about.html index e0595232..ed679ffa 100644 --- a/app/templates/about.html +++ b/app/templates/about.html @@ -8,10 +8,12 @@

{{ _('About %(site_name)s', site_name=g.site.name) }}

- -

{{g.site.name}} is a pyfedi instance created on {{instance.created_at}}. It is home to {{user_amount}} users, {{community_amount}} communities who discussed {{domains_amount}} domains. This instance is administerred and staffed by $PLACEHOLDER_ADMINS and $PLACEHOLDER_STAFF.

+

{{g.site.name}} is a pyfedi instance created on {{instance.created_at.strftime('%d-%m-%Y')}}. It is home to {{user_amount}} users (of which {{mau}} active in the last month). In the {{community_amount}} communities we discussed {{domains_amount}} domains and made {{posts_amount}} posts.

+

Team

+

This instance is administerred by {% for admin in admins %}{{ admin.user_name }}{{ ", " if not loop.last }}{% endfor %}.

+

It is moderated by {% for s in staff %}{{ s.user_name }}{{ ", " if not loop.last }}{% endfor %}.

Contact

-

Placeholder Admin email

+

$PLACEHOLDER_EMAIL

About Us

{{g.site.description | safe}}

{{g.site.sidebar}}

From a9f4fff576a58599bfa6f96cc4b8748bda8e9865 Mon Sep 17 00:00:00 2001 From: rimu <3310831+rimu@users.noreply.github.com> Date: Sun, 14 Apr 2024 21:49:42 +1200 Subject: [PATCH 02/24] also delete replies to deleted comments --- app/models.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/app/models.py b/app/models.py index 30f2506a..b96264f7 100644 --- a/app/models.py +++ b/app/models.py @@ -838,9 +838,11 @@ class User(UserMixin, db.Model): post.delete_dependencies() post.flush_cache() db.session.delete(post) + db.session.commit() post_replies = PostReply.query.filter_by(user_id=self.id).all() for reply in post_replies: - reply.body = reply.body_html = '' + reply.delete_dependencies() + db.session.delete(reply) db.session.commit() def mention_tag(self): @@ -1018,6 +1020,10 @@ class PostReply(db.Model): return parent.author.profile_id() def delete_dependencies(self): + for child_reply in self.child_replies(): + child_reply.delete_dependencies() + db.session.delete(child_reply) + db.session.query(Report).filter(Report.suspect_post_reply_id == self.id).delete() db.session.execute(text('DELETE FROM post_reply_vote WHERE post_reply_id = :post_reply_id'), {'post_reply_id': self.id}) @@ -1025,6 +1031,9 @@ class PostReply(db.Model): file = File.query.get(self.image_id) file.delete_from_disk() + def child_replies(self): + return PostReply.query.filter_by(parent_id=self.id).all() + def has_replies(self): reply = PostReply.query.filter_by(parent_id=self.id).first() return reply is not None From b193f715270949f707ac2f116b9e3539288fd500 Mon Sep 17 00:00:00 2001 From: rimu <3310831+rimu@users.noreply.github.com> Date: Mon, 15 Apr 2024 12:08:04 +1200 Subject: [PATCH 03/24] make spicyness of hot algo configurable through .env --- app/activitypub/util.py | 10 +++++----- app/post/routes.py | 10 +++++----- config.py | 4 ++++ 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/app/activitypub/util.py b/app/activitypub/util.py index fd86d652..1a463a79 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -1039,9 +1039,9 @@ def downvote_post(post, user): post.down_votes += 1 # Make 'hot' sort more spicy by amplifying the effect of early downvotes if post.up_votes + post.down_votes <= 30: - post.score -= 5.0 + post.score -= current_app.config['SPICY_UNDER_30'] elif post.up_votes + post.down_votes <= 60: - post.score -= 2.0 + post.score -= current_app.config['SPICY_UNDER_60'] else: post.score -= 1.0 vote = PostVote(user_id=user.id, post_id=post.id, author_id=post.author.id, @@ -1148,11 +1148,11 @@ def upvote_post(post, user): # Make 'hot' sort more spicy by amplifying the effect of early upvotes spicy_effect = effect if post.up_votes + post.down_votes <= 10: - spicy_effect = effect * 10 + spicy_effect = effect * current_app.config['SPICY_UNDER_10'] elif post.up_votes + post.down_votes <= 30: - spicy_effect = effect * 5 + spicy_effect = effect * current_app.config['SPICY_UNDER_30'] elif post.up_votes + post.down_votes <= 60: - spicy_effect = effect * 2 + spicy_effect = effect * current_app.config['SPICY_UNDER_60'] existing_vote = PostVote.query.filter_by(user_id=user.id, post_id=post.id).first() if not existing_vote: post.up_votes += 1 diff --git a/app/post/routes.py b/app/post/routes.py index f80ab8b3..9a820eab 100644 --- a/app/post/routes.py +++ b/app/post/routes.py @@ -305,20 +305,20 @@ def post_vote(post_id: int, vote_direction): post.up_votes += 1 # Make 'hot' sort more spicy by amplifying the effect of early upvotes if post.up_votes + post.down_votes <= 10: - post.score += 10 + post.score += current_app.config['SPICY_UNDER_10'] elif post.up_votes + post.down_votes <= 30: - post.score += 5 + post.score += current_app.config['SPICY_UNDER_30'] elif post.up_votes + post.down_votes <= 60: - post.score += 2 + post.score += current_app.config['SPICY_UNDER_60'] else: post.score += 1 else: effect = -1 post.down_votes += 1 if post.up_votes + post.down_votes <= 30: - post.score -= 5 + post.score -= current_app.config['SPICY_UNDER_30'] elif post.up_votes + post.down_votes <= 60: - post.score -= 2 + post.score -= current_app.config['SPICY_UNDER_60'] else: post.score -= 1 vote = PostVote(user_id=current_user.id, post_id=post.id, author_id=post.author.id, diff --git a/config.py b/config.py index 7dbe2cc4..67421320 100644 --- a/config.py +++ b/config.py @@ -49,3 +49,7 @@ class Config(object): CLOUDFLARE_API_TOKEN = os.environ.get('CLOUDFLARE_API_TOKEN') or '' CLOUDFLARE_ZONE_ID = os.environ.get('CLOUDFLARE_ZONE_ID') or '' + + SPICY_UNDER_10 = int(os.environ.get('SPICY_UNDER_10')) or 1 + SPICY_UNDER_30 = int(os.environ.get('SPICY_UNDER_30')) or 1 + SPICY_UNDER_60 = int(os.environ.get('SPICY_UNDER_60')) or 1 From 3c006332e3632847772004874153fe79355d4563 Mon Sep 17 00:00:00 2001 From: rimu <3310831+rimu@users.noreply.github.com> Date: Mon, 15 Apr 2024 12:16:29 +1200 Subject: [PATCH 04/24] site admins can delete comments --- app/post/routes.py | 2 +- app/templates/post/post_reply_options.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/post/routes.py b/app/post/routes.py index 9a820eab..fab5fcd7 100644 --- a/app/post/routes.py +++ b/app/post/routes.py @@ -1442,7 +1442,7 @@ def post_reply_delete(post_id: int, comment_id: int): post = Post.query.get_or_404(post_id) post_reply = PostReply.query.get_or_404(comment_id) community = post.community - if post_reply.user_id == current_user.id or community.is_moderator(): + if post_reply.user_id == current_user.id or community.is_moderator() or current_user.is_admin(): if post_reply.has_replies(): post_reply.body = 'Deleted by author' if post_reply.author.id == current_user.id else 'Deleted by moderator' post_reply.body_html = markdown_to_html(post_reply.body) diff --git a/app/templates/post/post_reply_options.html b/app/templates/post/post_reply_options.html index 9c77eed4..f2d99104 100644 --- a/app/templates/post/post_reply_options.html +++ b/app/templates/post/post_reply_options.html @@ -13,7 +13,7 @@
{{ _('Options for comment on "%(post_title)s"', post_title=post.title) }}
diff --git a/app/templates/community/add_video_post.html b/app/templates/community/add_video_post.html new file mode 100644 index 00000000..31dca5d3 --- /dev/null +++ b/app/templates/community/add_video_post.html @@ -0,0 +1,98 @@ +{% if theme() and file_exists('app/templates/themes/' + theme() + '/base.html') %} + {% extends 'themes/' + theme() + '/base.html' %} +{% else %} + {% extends "base.html" %} +{% endif %} %} +{% from 'bootstrap/form.html' import render_field %} + +{% block app_content %} +
+
+

{{ _('Create post') }}

+
+ {{ form.csrf_token() }} +
+ + +
+ {{ render_field(form.communities) }} + + {{ render_field(form.video_title) }} + {{ render_field(form.video_url) }} +

{{ _('Provide a URL ending with .mp4 or .webm.') }}

+ {{ render_field(form.video_body) }} + {% if not low_bandwidth %} + {% if markdown_editor %} + + {% else %} + + {% endif %} + {% endif %} + +
+
+ {{ render_field(form.notify_author) }} +
+
+ {{ render_field(form.sticky) }} +
+
+ {{ render_field(form.nsfw) }} +
+
+ {{ render_field(form.nsfl) }} +
+
+ +
+
+ + {{ render_field(form.submit) }} +
+
+ + +
+{% endblock %} \ No newline at end of file diff --git a/app/templates/post/_post_full.html b/app/templates/post/_post_full.html index 891700f3..f88c5e8f 100644 --- a/app/templates/post/_post_full.html +++ b/app/templates/post/_post_full.html @@ -92,6 +92,27 @@ {% endif %}

+ {% elif post.url.startswith('https://streamable.com') %} +
+ {% elif post.url.startswith('https://www.redgifs.com/watch/') %} +
+ {% endif %} + {% if 'youtube.com' in post.url %} +

{{ _('Watch on piped.video') }}

+
+ {% endif %} + {% elif post.type == POST_TYPE_VIDEO %} +

{{ post.url|shorten_url }} +

+ {% if post.url.endswith('.mp4') or post.url.endswith('.webm') %} +

+

{% endif %} {% if 'youtube.com' in post.url %}

{{ _('Watch on piped.video') }}

diff --git a/app/templates/post/_post_teaser.html b/app/templates/post/_post_teaser.html index 633053a3..36c201ff 100644 --- a/app/templates/post/_post_teaser.html +++ b/app/templates/post/_post_teaser.html @@ -16,15 +16,18 @@ {% if post.image_id %} {% else %} - {% if post.type == POST_TYPE_LINK and post.domain_id %} + {% if (post.type == POST_TYPE_LINK or post.type == POST_TYPE_VIDEO) and post.domain_id %} @@ -47,8 +50,8 @@ {% endif %}

{% if post.sticky %}{% endif %}{{ post.title }} {% if post.type == POST_TYPE_IMAGE %}{% endif %} - {% if post.type == POST_TYPE_LINK and post.domain_id %} - {% if post.url and 'youtube.com' in post.url %} + {% if (post.type == POST_TYPE_LINK or post.type == POST_TYPE_VIDEO) and post.domain_id %} + {% if post.url and (post.type == POST_TYPE_VIDEO or 'youtube.com' in post.url) %} {% elif post.url.endswith('.mp3') %} diff --git a/app/templates/post/_post_teaser_masonry.html b/app/templates/post/_post_teaser_masonry.html index b84f6ded..7dbedb91 100644 --- a/app/templates/post/_post_teaser_masonry.html +++ b/app/templates/post/_post_teaser_masonry.html @@ -11,7 +11,7 @@ {% set thumbnail = post.image.view_url() %} {% endif %}
- {% if post.type == POST_TYPE_LINK %} + {% if post.type == POST_TYPE_LINK or post.type == POST_TYPE_VIDEO %} {% if post.image.medium_url() %} {{ post.image.alt_text if post.image.alt_text else '' }} +
+

{{ _('Edit post') }}

+
+ {{ form.csrf_token() }} + {{ render_field(form.video_title) }} + {{ render_field(form.video_url) }} + {{ render_field(form.video_body) }} + {% if markdown_editor %} + + {% endif %} +
+
+ {{ render_field(form.notify_author) }} +
+
+ {{ render_field(form.sticky) }} +
+
+ {{ render_field(form.nsfw) }} +
+
+ {{ render_field(form.nsfl) }} +
+ +
+ +
+
+ + {{ render_field(form.submit) }} +
+
+ + + +
+{% endblock %} \ No newline at end of file diff --git a/app/topic/routes.py b/app/topic/routes.py index 0751bcb4..82379cc7 100644 --- a/app/topic/routes.py +++ b/app/topic/routes.py @@ -9,7 +9,7 @@ from flask_babel import _ from sqlalchemy import text, desc, or_ from app.activitypub.signature import post_request -from app.constants import SUBSCRIPTION_NONMEMBER, POST_TYPE_IMAGE, POST_TYPE_LINK +from app.constants import SUBSCRIPTION_NONMEMBER, POST_TYPE_IMAGE, POST_TYPE_LINK, POST_TYPE_VIDEO from app.inoculation import inoculation from app.models import Topic, Community, Post, utcnow, CommunityMember, CommunityJoinRequest, User from app.topic import bp @@ -117,7 +117,8 @@ def show_topic(topic_path): show_post_community=True, moderating_communities=moderating_communities(current_user.get_id()), joined_communities=joined_communities(current_user.get_id()), inoculation=inoculation[randint(0, len(inoculation) - 1)], - POST_TYPE_LINK=POST_TYPE_LINK, POST_TYPE_IMAGE=POST_TYPE_IMAGE) + POST_TYPE_LINK=POST_TYPE_LINK, POST_TYPE_IMAGE=POST_TYPE_IMAGE, + POST_TYPE_VIDEO=POST_TYPE_VIDEO) else: abort(404) diff --git a/app/utils.py b/app/utils.py index c354b085..59fc23cd 100644 --- a/app/utils.py +++ b/app/utils.py @@ -168,6 +168,13 @@ def is_image_url(url): return any(path.endswith(extension) for extension in common_image_extensions) +def is_video_url(url): + parsed_url = urlparse(url) + path = parsed_url.path.lower() + common_video_extensions = ['.mp4', '.webm'] + return any(path.endswith(extension) for extension in common_video_extensions) + + # sanitise HTML using an allow list def allowlist_html(html: str) -> str: if html is None or html == '': diff --git a/pyfedi.py b/pyfedi.py index 0d1b4b4b..6b0f8a6d 100644 --- a/pyfedi.py +++ b/pyfedi.py @@ -8,7 +8,7 @@ from flask_login import current_user from app import create_app, db, cli import os, click from flask import session, g, json, request, current_app -from app.constants import POST_TYPE_LINK, POST_TYPE_IMAGE, POST_TYPE_ARTICLE +from app.constants import POST_TYPE_LINK, POST_TYPE_IMAGE, POST_TYPE_ARTICLE, POST_TYPE_VIDEO from app.models import Site from app.utils import getmtime, gibberish, shorten_string, shorten_url, digits, user_access, community_membership, \ can_create_post, can_upvote, can_downvote, shorten_number, ap_datetime, current_theme, community_link_to_href, \ @@ -22,7 +22,8 @@ cli.register(app) def app_context_processor(): def getmtime(filename): return os.path.getmtime('app/static/' + filename) - return dict(getmtime=getmtime, post_type_link=POST_TYPE_LINK, post_type_image=POST_TYPE_IMAGE, post_type_article=POST_TYPE_ARTICLE) + return dict(getmtime=getmtime, post_type_link=POST_TYPE_LINK, post_type_image=POST_TYPE_IMAGE, + post_type_article=POST_TYPE_ARTICLE, post_type_video=POST_TYPE_VIDEO) @app.shell_context_processor From 98207edb138c04a576d157dcb2ecf872a6f443e2 Mon Sep 17 00:00:00 2001 From: rimu <3310831+rimu@users.noreply.github.com> Date: Tue, 16 Apr 2024 21:16:10 +1200 Subject: [PATCH 16/24] migration for tags and languages --- app/models.py | 27 ++++++++ .../fd2af23f4b1f_tags_and_languages.py | 69 +++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 migrations/versions/fd2af23f4b1f_tags_and_languages.py diff --git a/app/models.py b/app/models.py index b96264f7..6a174cff 100644 --- a/app/models.py +++ b/app/models.py @@ -1294,6 +1294,33 @@ class Site(db.Model): return User.query.filter_by(deleted=False, banned=False).join(user_role).filter(user_role.c.role_id == 4).all() +class Tag(db.Model): + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.String(256)) + + +class Language(db.Model): + id = db.Column(db.Integer, primary_key=True) + code = db.Column(db.String(5), index=True) + name = db.Column(db.String(50)) + + +post_language = db.Table('post_language', db.Column('post_id', db.Integer, db.ForeignKey('post.id')), + db.Column('language_id', db.Integer, db.ForeignKey('language.id')), + db.PrimaryKeyConstraint('post_id', 'language_id') + ) + +community_language = db.Table('community_language', db.Column('community_id', db.Integer, db.ForeignKey('community.id')), + db.Column('language_id', db.Integer, db.ForeignKey('language.id')), + db.PrimaryKeyConstraint('community_id', 'language_id') + ) + +post_tag = db.Table('post_tag', db.Column('post_id', db.Integer, db.ForeignKey('post.id')), + db.Column('tag_id', db.Integer, db.ForeignKey('tag.id')), + db.PrimaryKeyConstraint('post_id', 'tag_id') + ) + + @login.user_loader def load_user(id): return User.query.get(int(id)) diff --git a/migrations/versions/fd2af23f4b1f_tags_and_languages.py b/migrations/versions/fd2af23f4b1f_tags_and_languages.py new file mode 100644 index 00000000..6c5e0a00 --- /dev/null +++ b/migrations/versions/fd2af23f4b1f_tags_and_languages.py @@ -0,0 +1,69 @@ +"""tags and languages + +Revision ID: fd2af23f4b1f +Revises: 91a931afd6d9 +Create Date: 2024-04-16 21:15:07.225254 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'fd2af23f4b1f' +down_revision = '91a931afd6d9' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('language', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('code', sa.String(length=5), nullable=True), + sa.Column('name', sa.String(length=50), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + with op.batch_alter_table('language', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_language_code'), ['code'], unique=False) + + op.create_table('tag', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=256), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('community_language', + sa.Column('community_id', sa.Integer(), nullable=False), + sa.Column('language_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['community_id'], ['community.id'], ), + sa.ForeignKeyConstraint(['language_id'], ['language.id'], ), + sa.PrimaryKeyConstraint('community_id', 'language_id') + ) + op.create_table('post_language', + sa.Column('post_id', sa.Integer(), nullable=False), + sa.Column('language_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['language_id'], ['language.id'], ), + sa.ForeignKeyConstraint(['post_id'], ['post.id'], ), + sa.PrimaryKeyConstraint('post_id', 'language_id') + ) + op.create_table('post_tag', + sa.Column('post_id', sa.Integer(), nullable=False), + sa.Column('tag_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['post_id'], ['post.id'], ), + sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], ), + sa.PrimaryKeyConstraint('post_id', 'tag_id') + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('post_tag') + op.drop_table('post_language') + op.drop_table('community_language') + op.drop_table('tag') + with op.batch_alter_table('language', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_language_code')) + + op.drop_table('language') + # ### end Alembic commands ### From 219b22e34a72e209d77b7e5a9b569a42f0bd8c7b Mon Sep 17 00:00:00 2001 From: rimu <3310831+rimu@users.noreply.github.com> Date: Tue, 16 Apr 2024 21:23:19 +1200 Subject: [PATCH 17/24] migration post one language --- app/models.py | 52 +++++++++++++++++++++++++-------------------------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/app/models.py b/app/models.py index 6a174cff..07913593 100644 --- a/app/models.py +++ b/app/models.py @@ -169,6 +169,28 @@ class ChatMessage(db.Model): sender = db.relationship('User', foreign_keys=[sender_id]) +class Tag(db.Model): + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.String(256)) + + +class Language(db.Model): + id = db.Column(db.Integer, primary_key=True) + code = db.Column(db.String(5), index=True) + name = db.Column(db.String(50)) + + +community_language = db.Table('community_language', db.Column('community_id', db.Integer, db.ForeignKey('community.id')), + db.Column('language_id', db.Integer, db.ForeignKey('language.id')), + db.PrimaryKeyConstraint('community_id', 'language_id') + ) + +post_tag = db.Table('post_tag', db.Column('post_id', db.Integer, db.ForeignKey('post.id')), + db.Column('tag_id', db.Integer, db.ForeignKey('tag.id')), + db.PrimaryKeyConstraint('post_id', 'tag_id') + ) + + class File(db.Model): id = db.Column(db.Integer, primary_key=True) file_path = db.Column(db.String(255)) @@ -365,6 +387,7 @@ class Community(db.Model): replies = db.relationship('PostReply', lazy='dynamic', cascade="all, delete-orphan") icon = db.relationship('File', foreign_keys=[icon_id], single_parent=True, backref='community', cascade="all, delete-orphan") image = db.relationship('File', foreign_keys=[image_id], single_parent=True, cascade="all, delete-orphan") + languages = db.relationship('Language', lazy='dynamic', secondary=community_language, backref=db.backref('communities', lazy='dynamic')) @cache.memoize(timeout=500) def icon_image(self, size='default') -> str: @@ -895,7 +918,9 @@ class Post(db.Model): language = db.Column(db.String(10)) edited_at = db.Column(db.DateTime) reports = db.Column(db.Integer, default=0) # how many times this post has been reported. Set to -1 to ignore reports + language_id = db.Column(db.Integer, index=True) cross_posts = db.Column(MutableList.as_mutable(ARRAY(db.Integer))) + tags = db.relationship('Tag', lazy='dynamic', secondary=post_tag, backref=db.backref('posts', lazy='dynamic')) ap_id = db.Column(db.String(255), index=True) ap_create_id = db.Column(db.String(100)) @@ -1294,33 +1319,6 @@ class Site(db.Model): return User.query.filter_by(deleted=False, banned=False).join(user_role).filter(user_role.c.role_id == 4).all() -class Tag(db.Model): - id = db.Column(db.Integer, primary_key=True) - name = db.Column(db.String(256)) - - -class Language(db.Model): - id = db.Column(db.Integer, primary_key=True) - code = db.Column(db.String(5), index=True) - name = db.Column(db.String(50)) - - -post_language = db.Table('post_language', db.Column('post_id', db.Integer, db.ForeignKey('post.id')), - db.Column('language_id', db.Integer, db.ForeignKey('language.id')), - db.PrimaryKeyConstraint('post_id', 'language_id') - ) - -community_language = db.Table('community_language', db.Column('community_id', db.Integer, db.ForeignKey('community.id')), - db.Column('language_id', db.Integer, db.ForeignKey('language.id')), - db.PrimaryKeyConstraint('community_id', 'language_id') - ) - -post_tag = db.Table('post_tag', db.Column('post_id', db.Integer, db.ForeignKey('post.id')), - db.Column('tag_id', db.Integer, db.ForeignKey('tag.id')), - db.PrimaryKeyConstraint('post_id', 'tag_id') - ) - - @login.user_loader def load_user(id): return User.query.get(int(id)) From 8be961bea8b3f8a7a3529a66ecbaa831dff8ec95 Mon Sep 17 00:00:00 2001 From: rimu <3310831+rimu@users.noreply.github.com> Date: Tue, 16 Apr 2024 21:49:05 +1200 Subject: [PATCH 18/24] save language of posts and communities --- app/activitypub/util.py | 25 ++++++++++- app/community/routes.py | 2 +- .../980966fba5f4_post_one_language.py | 42 +++++++++++++++++++ 3 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 migrations/versions/980966fba5f4_post_one_language.py diff --git a/app/activitypub/util.py b/app/activitypub/util.py index a93357b8..e110608b 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -12,7 +12,8 @@ from flask_babel import _ from sqlalchemy import text, func from app import db, cache, constants, celery from app.models import User, Post, Community, BannedInstances, File, PostReply, AllowedInstances, Instance, utcnow, \ - PostVote, PostReplyVote, ActivityPubLog, Notification, Site, CommunityMember, InstanceRole, Report, Conversation + PostVote, PostReplyVote, ActivityPubLog, Notification, Site, CommunityMember, InstanceRole, Report, Conversation, \ + Language import time import base64 import requests @@ -342,6 +343,16 @@ def find_actor_or_create(actor: str, create_if_not_found=True, community_only=Fa return None +def find_language_or_create(code: str, name: str) -> Language: + existing_language = Language.query.filter(Language.code == code).first() + if existing_language: + return existing_language + else: + new_language = Language(code=code, name=name) + db.session.add(new_language) + return new_language + + def extract_domain_and_actor(url_string: str): # Parse the URL parsed_url = urlparse(url_string) @@ -637,6 +648,9 @@ def actor_json_to_model(activity_json, address, server): image = File(source_url=activity_json['image']['url']) community.image = image db.session.add(image) + if 'language' in activity_json and isinstance(activity_json['language'], list): + for ap_language in activity_json['language']: + community.languages.append(find_language_or_create(ap_language['identifier'], ap_language['name'])) db.session.add(community) db.session.commit() if community.icon_id: @@ -1457,6 +1471,9 @@ def create_post(activity_log: ActivityPubLog, community: Community, request_json else: post = None activity_log.exception_message = domain.name + ' is blocked by admin' + if 'language' in request_json['object'] and isinstance(request_json['object']['language'], dict): + language = find_language_or_create(request_json['object']['language']['identifier'], request_json['object']['language']['name']) + post.language_id = language.id if post is not None: if 'image' in request_json['object'] and post.image is None: image = File(source_url=request_json['object']['image']['url']) @@ -1541,6 +1558,11 @@ def update_post_from_activity(post: Post, request_json: dict): name += ' ' + microblog_content_to_title(post.body_html) nsfl_in_title = '[NSFL]' in name.upper() or '(NSFL)' in name.upper() post.title = name + # Language + if 'language' in request_json['object'] and isinstance(request_json['object']['language'], dict): + language = find_language_or_create(request_json['object']['language']['identifier'], request_json['object']['language']['name']) + post.language_id = language.id + # Links old_url = post.url old_image_id = post.image_id post.url = '' @@ -1599,6 +1621,7 @@ def update_post_from_activity(post: Post, request_json: dict): else: post.url = old_url # don't change if url changed from non-banned domain to banned domain + # Posts which link to the same url as other posts new_cross_posts = Post.query.filter(Post.id != post.id, Post.url == post.url, Post.posted_at > utcnow() - timedelta(days=6)).all() for ncp in new_cross_posts: diff --git a/app/community/routes.py b/app/community/routes.py index 94c296e9..63d41c0a 100644 --- a/app/community/routes.py +++ b/app/community/routes.py @@ -10,7 +10,7 @@ from sqlalchemy import or_, desc, text from app import db, constants, cache from app.activitypub.signature import RsaKeys, post_request -from app.activitypub.util import default_context, notify_about_post, find_actor_or_create, make_image_sizes +from app.activitypub.util import default_context, notify_about_post, make_image_sizes from app.chat.util import send_message from app.community.forms import SearchRemoteCommunity, CreateDiscussionForm, CreateImageForm, CreateLinkForm, \ ReportCommunityForm, \ diff --git a/migrations/versions/980966fba5f4_post_one_language.py b/migrations/versions/980966fba5f4_post_one_language.py new file mode 100644 index 00000000..bc302396 --- /dev/null +++ b/migrations/versions/980966fba5f4_post_one_language.py @@ -0,0 +1,42 @@ +"""post one language + +Revision ID: 980966fba5f4 +Revises: fd2af23f4b1f +Create Date: 2024-04-16 21:23:34.642869 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '980966fba5f4' +down_revision = 'fd2af23f4b1f' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('post_language') + with op.batch_alter_table('post', schema=None) as batch_op: + batch_op.add_column(sa.Column('language_id', sa.Integer(), nullable=True)) + batch_op.create_index(batch_op.f('ix_post_language_id'), ['language_id'], unique=False) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('post', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_post_language_id')) + batch_op.drop_column('language_id') + + op.create_table('post_language', + sa.Column('post_id', sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column('language_id', sa.INTEGER(), autoincrement=False, nullable=False), + sa.ForeignKeyConstraint(['language_id'], ['language.id'], name='post_language_language_id_fkey'), + sa.ForeignKeyConstraint(['post_id'], ['post.id'], name='post_language_post_id_fkey'), + sa.PrimaryKeyConstraint('post_id', 'language_id', name='post_language_pkey') + ) + # ### end Alembic commands ### From a4f0899e680bad23832ee889dfc109be44fe6076 Mon Sep 17 00:00:00 2001 From: rra Date: Tue, 16 Apr 2024 11:56:55 +0200 Subject: [PATCH 19/24] fallback to default value if env variable not set --- config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config.py b/config.py index 687a8fc2..49b9d109 100644 --- a/config.py +++ b/config.py @@ -50,6 +50,6 @@ class Config(object): CLOUDFLARE_API_TOKEN = os.environ.get('CLOUDFLARE_API_TOKEN') or '' CLOUDFLARE_ZONE_ID = os.environ.get('CLOUDFLARE_ZONE_ID') or '' - SPICY_UNDER_10 = float(os.environ.get('SPICY_UNDER_10')) or 1.0 - SPICY_UNDER_30 = float(os.environ.get('SPICY_UNDER_30')) or 1.0 - SPICY_UNDER_60 = float(os.environ.get('SPICY_UNDER_60')) or 1.0 + SPICY_UNDER_10 = float(os.environ.get('SPICY_UNDER_10', 1.0)) + SPICY_UNDER_30 = float(os.environ.get('SPICY_UNDER_30', 1.0)) + SPICY_UNDER_60 = float(os.environ.get('SPICY_UNDER_60', 1.0)) From 2bf9b4a8c8419054a48036a1746c2c496062853e Mon Sep 17 00:00:00 2001 From: rra Date: Tue, 16 Apr 2024 12:10:24 +0200 Subject: [PATCH 20/24] instance contact email interface, forms and display on about page --- app/admin/forms.py | 3 ++- app/admin/routes.py | 1 + app/templates/about.html | 8 ++++---- app/templates/admin/site.html | 1 + 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/app/admin/forms.py b/app/admin/forms.py index 43c48e51..59abaac6 100644 --- a/app/admin/forms.py +++ b/app/admin/forms.py @@ -1,7 +1,7 @@ from flask_wtf import FlaskForm from flask_wtf.file import FileRequired, FileAllowed from sqlalchemy import func -from wtforms import StringField, PasswordField, SubmitField, HiddenField, BooleanField, TextAreaField, SelectField, \ +from wtforms import StringField, PasswordField, SubmitField, EmailField, HiddenField, BooleanField, TextAreaField, SelectField, \ FileField, IntegerField from wtforms.validators import ValidationError, DataRequired, Email, EqualTo, Length, Optional from flask_babel import _, lazy_gettext as _l @@ -17,6 +17,7 @@ class SiteProfileForm(FlaskForm): ]) sidebar = TextAreaField(_l('Sidebar')) legal_information = TextAreaField(_l('Legal information')) + contact_email = EmailField(_l('General instance contact email address'), validators=[Email(), DataRequired(), Length(min=5, max=255)]) submit = SubmitField(_l('Save')) diff --git a/app/admin/routes.py b/app/admin/routes.py index eba8418e..2d3d2030 100644 --- a/app/admin/routes.py +++ b/app/admin/routes.py @@ -48,6 +48,7 @@ def admin_site(): site.sidebar = form.sidebar.data site.legal_information = form.legal_information.data site.updated = utcnow() + site.contact_email = form.contact_email.data if site.id is None: db.session.add(site) db.session.commit() diff --git a/app/templates/about.html b/app/templates/about.html index ed679ffa..a7a7c018 100644 --- a/app/templates/about.html +++ b/app/templates/about.html @@ -13,13 +13,13 @@

This instance is administerred by {% for admin in admins %}{{ admin.user_name }}{{ ", " if not loop.last }}{% endfor %}.

It is moderated by {% for s in staff %}{{ s.user_name }}{{ ", " if not loop.last }}{% endfor %}.

Contact

-

$PLACEHOLDER_EMAIL

+

{{g.site.contact_email | safe }}

About Us

-

{{g.site.description | safe}}

-

{{g.site.sidebar}}

+

{{g.site.description | safe }}

+

{{g.site.sidebar | safe }}

{% if g.site.legal_information %}

Legal Information

-

{{g.site.legal_information}}

+

{{g.site.legal_information | safe }}

Our Privacy Policy

{% endif %}

diff --git a/app/templates/admin/site.html b/app/templates/admin/site.html index 31bfa7be..57496d52 100644 --- a/app/templates/admin/site.html +++ b/app/templates/admin/site.html @@ -20,6 +20,7 @@

HTML is allowed in this field.

{{ render_field(form.legal_information) }}

HTML is allowed in this field.

+ {{ render_field(form.contact_email) }} {{ render_field(form.submit) }} From be4dbfb10762785ec772b2473f99203b0383b3af Mon Sep 17 00:00:00 2001 From: rra Date: Tue, 16 Apr 2024 14:34:22 +0200 Subject: [PATCH 21/24] add gunicorn install to production docs --- INSTALL.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/INSTALL.md b/INSTALL.md index 1c896b18..40902d35 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -214,6 +214,13 @@ Once you have ngrok working, edit the `.env` file and change the `SERVER_NAME` v ## Running PieFed in production +Running PieFed in production relies on several additional packages that need to be installed. + +```bash +source venv/bin/activate #if not already in virtual environment +pip3 install gunicorn celery +``` + Copy `celery_worker.default.py` to `celery_worker.py`. Edit `DATABASE_URL` and `SERVER_NAME` to have the same values as in `.env`. Edit `gunicorn.conf.py` and change `worker_tmp_dir` if needed. From 7e9f447c708b708e77fe8680405c6c914d5b9c55 Mon Sep 17 00:00:00 2001 From: rra Date: Tue, 16 Apr 2024 18:29:27 +0200 Subject: [PATCH 22/24] minor fixes to theming engine --- app/templates/auth/login.html | 6 +++++- app/templates/base.html | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/app/templates/auth/login.html b/app/templates/auth/login.html index b37e5712..cb6c2a91 100644 --- a/app/templates/auth/login.html +++ b/app/templates/auth/login.html @@ -1,4 +1,8 @@ -{% extends 'base.html' %} +{% if theme() and file_exists('app/templates/themes/' + theme() + '/base.html') %} + {% extends 'themes/' + theme() + '/base.html' %} +{% else %} + {% extends "base.html" %} +{% endif %} %} {% from 'bootstrap/form.html' import render_form %} {% block app_content %} diff --git a/app/templates/base.html b/app/templates/base.html index df9183fe..37a58779 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -270,7 +270,7 @@ {% endif %} {% if theme() and file_exists('app/templates/themes/' + theme() + '/scripts.js') %} - {% endif %} {% block end_scripts %} From 4c48f72dcdbc8347146ef79382e3385e77647e94 Mon Sep 17 00:00:00 2001 From: rimu <3310831+rimu@users.noreply.github.com> Date: Wed, 17 Apr 2024 19:17:11 +1200 Subject: [PATCH 23/24] video avatars #58 see https://piefed.social/u/const_void@lemmy.ml --- app/main/routes.py | 1 + app/templates/post/continue_discussion.html | 2 +- app/templates/post/post.html | 2 +- app/templates/user/show_profile.html | 12 +++++++++++- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/app/main/routes.py b/app/main/routes.py index 2bde90bd..bdc8a796 100644 --- a/app/main/routes.py +++ b/app/main/routes.py @@ -301,6 +301,7 @@ def list_files(directory): @bp.route('/test') def test(): + x = find_actor_or_create('https://lemmy.ml/u/const_void') md = "::: spoiler I'm all for ya having fun and your right to hurt yourself.\n\nI am a former racer, commuter, and professional Buyer for a chain of bike shops. I'm also disabled from the crash involving the 6th and 7th cars that have hit me in the last 170k+ miles of riding. I only barely survived what I simplify as a \"broken neck and back.\" Cars making U-turns are what will get you if you ride long enough, \n\nespecially commuting. It will look like just another person turning in front of you, you'll compensate like usual, and before your brain can even register what is really happening, what was your normal escape route will close and you're going to crash really hard. It is the only kind of crash that your intuition is useless against.\n:::" return markdown_to_html(md) diff --git a/app/templates/post/continue_discussion.html b/app/templates/post/continue_discussion.html index d00dca29..d7f760ac 100644 --- a/app/templates/post/continue_discussion.html +++ b/app/templates/post/continue_discussion.html @@ -21,7 +21,7 @@ {% else %} {% if comment['comment'].author.avatar_id and comment['comment'].score > -10 and not low_bandwidth %} - Avatar + Avatar {% endif %} {{ comment['comment'].author.display_name() }} diff --git a/app/templates/post/post.html b/app/templates/post/post.html index 04006125..2caa0f4a 100644 --- a/app/templates/post/post.html +++ b/app/templates/post/post.html @@ -82,7 +82,7 @@ {% else %} {% if comment['comment'].author.avatar_id and comment['comment'].score > -10 and not low_bandwidth %} - + {% endif %} {{ comment['comment'].author.display_name() }} diff --git a/app/templates/user/show_profile.html b/app/templates/user/show_profile.html index b6d17c05..ea8d6dfe 100644 --- a/app/templates/user/show_profile.html +++ b/app/templates/user/show_profile.html @@ -26,7 +26,17 @@ {% elif user.avatar_image() != '' %}
- {{ _('Profile pic') }} + {% if low_bandwidth %} + {{ _('Profile pic') }} + {% else %} + {% if user.avatar.source_url and user.avatar.source_url.endswith('.mp4') %} + + {% else %} + {{ _('Profile pic') }} + {% endif %} + {% endif %}

{{ user.display_name() if user.is_local() else user.display_name() + ', ' + user.ap_id }}

From a8f7ebf4429de4fa642dd00188a6ca84363d25d1 Mon Sep 17 00:00:00 2001 From: rimu <3310831+rimu@users.noreply.github.com> Date: Wed, 17 Apr 2024 19:42:36 +1200 Subject: [PATCH 24/24] streamable #147 --- app/community/routes.py | 8 ++++---- app/community/util.py | 22 ++++++++++++++++++---- app/templates/post/_post_full.html | 4 ++++ 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/app/community/routes.py b/app/community/routes.py index 63d41c0a..eb9c3b54 100644 --- a/app/community/routes.py +++ b/app/community/routes.py @@ -500,7 +500,7 @@ def add_discussion_post(actor): if not community.local_only: federate_post(community, post) - return redirect(f"/c/{community.link()}") + return redirect(f"/post/{post.id}") else: form.communities.data = community.id form.notify_author.data = True @@ -573,7 +573,7 @@ def add_image_post(actor): if not community.local_only: federate_post(community, post) - return redirect(f"/c/{community.link()}") + return redirect(f"/post/{post.id}") else: form.communities.data = community.id form.notify_author.data = True @@ -646,7 +646,7 @@ def add_link_post(actor): if not community.local_only: federate_post(community, post) - return redirect(f"/c/{community.link()}") + return redirect(f"/post/{post.id}") else: form.communities.data = community.id form.notify_author.data = True @@ -719,7 +719,7 @@ def add_video_post(actor): if not community.local_only: federate_post(community, post) - return redirect(f"/c/{community.link()}") + return redirect(f"/post/{post.id}") else: form.communities.data = community.id form.notify_author.data = True diff --git a/app/community/util.py b/app/community/util.py index 04326fee..80d88132 100644 --- a/app/community/util.py +++ b/app/community/util.py @@ -310,6 +310,7 @@ def save_post(form, post: Post, type: str): post.image = file db.session.add(file) elif type == 'video': + form.video_url.data = form.video_url.data.strip() post.title = form.video_title.data post.body = form.video_body.data post.body_html = markdown_to_html(post.body) @@ -324,10 +325,23 @@ def save_post(form, post: Post, type: str): if post.image_id: remove_old_file(post.image_id) post.image_id = None - - file = File(source_url=form.video_url.data) # make_image_sizes() will take care of turning this into a still image - post.image = file - db.session.add(file) + if form.video_url.data.endswith('.mp4') or form.video_url.data.endswith('.webm'): + file = File(source_url=form.video_url.data) # make_image_sizes() will take care of turning this into a still image + post.image = file + db.session.add(file) + else: + # check opengraph tags on the page and make a thumbnail if an image is available in the og:image meta tag + opengraph = opengraph_parse(form.video_url.data) + if opengraph and (opengraph.get('og:image', '') != '' or opengraph.get('og:image:url', '') != ''): + filename = opengraph.get('og:image') or opengraph.get('og:image:url') + filename_for_extension = filename.split('?')[0] if '?' in filename else filename + unused, file_extension = os.path.splitext(filename_for_extension) + if file_extension.lower() in allowed_extensions and not filename.startswith('/'): + file = url_to_thumbnail_file(filename) + if file: + file.alt_text = shorten_string(opengraph.get('og:title'), 295) + post.image = file + db.session.add(file) elif type == 'poll': ... diff --git a/app/templates/post/_post_full.html b/app/templates/post/_post_full.html index f88c5e8f..1304a4e5 100644 --- a/app/templates/post/_post_full.html +++ b/app/templates/post/_post_full.html @@ -113,6 +113,10 @@ {% endif %}

+ {% elif post.url.startswith('https://streamable.com') %} +
+ {% elif post.url.startswith('https://www.redgifs.com/watch/') %} +
{% endif %} {% if 'youtube.com' in post.url %}

{{ _('Watch on piped.video') }}