From 3bc30ec99c645d50fdc49a99190510a891f04985 Mon Sep 17 00:00:00 2001 From: rimu <3310831+rimu@users.noreply.github.com> Date: Wed, 8 May 2024 21:07:22 +1200 Subject: [PATCH] let admin override language for remote communities #51 --- app/activitypub/util.py | 2 +- app/admin/forms.py | 3 ++ app/admin/routes.py | 16 ++++++++-- app/admin/util.py | 5 ++- app/cli.py | 3 +- app/community/forms.py | 3 ++ app/community/routes.py | 21 ++++++++++-- app/community/util.py | 2 +- app/models.py | 5 +++ app/templates/admin/edit_community.html | 2 ++ app/templates/community/add_local.html | 1 + app/templates/community/community_edit.html | 1 + app/utils.py | 10 +++++- .../94828ddc7c63_community_remote_language.py | 32 +++++++++++++++++++ 14 files changed, 97 insertions(+), 9 deletions(-) create mode 100644 migrations/versions/94828ddc7c63_community_remote_language.py diff --git a/app/activitypub/util.py b/app/activitypub/util.py index 65f9f01c..a587975b 100644 --- a/app/activitypub/util.py +++ b/app/activitypub/util.py @@ -535,7 +535,7 @@ def refresh_community_profile_task(community_id): community.image = image db.session.add(image) cover_changed = True - if 'language' in activity_json and isinstance(activity_json['language'], list): + if 'language' in activity_json and isinstance(activity_json['language'], list) and not community.ignore_remote_language: for ap_language in activity_json['language']: new_language = find_language_or_create(ap_language['identifier'], ap_language['name']) if new_language not in community.languages: diff --git a/app/admin/forms.py b/app/admin/forms.py index e559777b..7e168130 100644 --- a/app/admin/forms.py +++ b/app/admin/forms.py @@ -3,6 +3,7 @@ from flask_wtf.file import FileRequired, FileAllowed from sqlalchemy import func from wtforms import StringField, PasswordField, SubmitField, EmailField, HiddenField, BooleanField, TextAreaField, SelectField, \ FileField, IntegerField +from wtforms.fields.choices import SelectMultipleField from wtforms.validators import ValidationError, DataRequired, Email, EqualTo, Length, Optional from flask_babel import _, lazy_gettext as _l @@ -83,6 +84,8 @@ class EditCommunityForm(FlaskForm): ('masonry_wide', _l('Wide masonry'))] default_layout = SelectField(_l('Layout'), coerce=str, choices=layouts, validators=[Optional()]) posting_warning = StringField(_l('Posting warning'), validators=[Optional(), Length(min=3, max=512)]) + languages = SelectMultipleField(_l('Languages'), coerce=int, validators=[Optional()], render_kw={'class': 'form-select'}) + ignore_remote_language = BooleanField(_l('Override remote language setting')) submit = SubmitField(_l('Save')) def validate(self, extra_validators=None): diff --git a/app/admin/routes.py b/app/admin/routes.py index cc252449..1aca82eb 100644 --- a/app/admin/routes.py +++ b/app/admin/routes.py @@ -18,10 +18,10 @@ from app.admin.util import unsubscribe_from_everything_then_delete, unsubscribe_ from app.community.util import save_icon_file, save_banner_file from app.constants import REPORT_STATE_NEW, REPORT_STATE_ESCALATED from app.models import AllowedInstances, BannedInstances, ActivityPubLog, utcnow, Site, Community, CommunityMember, \ - User, Instance, File, Report, Topic, UserRegistration, Role, Post, PostReply + User, Instance, File, Report, Topic, UserRegistration, Role, Post, PostReply, Language from app.utils import render_template, permission_required, set_setting, get_setting, gibberish, markdown_to_html, \ moderating_communities, joined_communities, finalize_user_setup, theme_list, blocked_phrases, blocked_referrers, \ - topic_tree + topic_tree, languages_for_form from app.admin import bp @@ -236,6 +236,7 @@ def admin_community_edit(community_id): form = EditCommunityForm() community = Community.query.get_or_404(community_id) form.topic.choices = topics_for_form(0) + form.languages.choices = languages_for_form() if form.validate_on_submit(): community.name = form.url.data community.title = form.title.data @@ -256,6 +257,7 @@ def admin_community_edit(community_id): community.topic_id = form.topic.data if form.topic.data != 0 else None community.default_layout = form.default_layout.data community.posting_warning = form.posting_warning.data + community.ignore_remote_language = form.ignore_remote_language.data icon_file = request.files['icon_file'] if icon_file and icon_file.filename != '': @@ -272,6 +274,14 @@ def admin_community_edit(community_id): if file: community.image = file + # Languages of the community + db.session.execute(text('DELETE FROM "community_language" WHERE community_id = :community_id'), + {'community_id': community_id}) + for language_choice in form.languages.data: + community.languages.append(Language.query.get(language_choice)) + # Always include the undetermined language, so posts with no language will be accepted + community.languages.append(Language.query.filter(Language.code == 'und').first()) + db.session.commit() if community.topic_id: community.topic.num_communities = community.topic.communities.count() @@ -298,6 +308,8 @@ def admin_community_edit(community_id): form.topic.data = community.topic_id if community.topic_id else None form.default_layout.data = community.default_layout form.posting_warning.data = community.posting_warning + form.languages.data = community.language_ids() + form.ignore_remote_language.data = community.ignore_remote_language return render_template('admin/edit_community.html', title=_('Edit community'), form=form, community=community, moderating_communities=moderating_communities(current_user.get_id()), joined_communities=joined_communities(current_user.get_id()), diff --git a/app/admin/util.py b/app/admin/util.py index 43b9be57..ca71bd10 100644 --- a/app/admin/util.py +++ b/app/admin/util.py @@ -7,7 +7,7 @@ from flask_babel import _ from app import db, cache, celery from app.activitypub.signature import post_request, default_context -from app.models import User, Community, Instance, Site, ActivityPubLog, CommunityMember, Topic +from app.models import User, Community, Instance, Site, ActivityPubLog, CommunityMember, Language from app.utils import gibberish, topic_tree @@ -124,3 +124,6 @@ def topics_for_form_children(topics, current_topic: int, depth: int) -> List[Tup if topic['children']: result.extend(topics_for_form_children(topic['children'], current_topic, depth + 1)) return result + + + diff --git a/app/cli.py b/app/cli.py index 612114e8..3dc5534e 100644 --- a/app/cli.py +++ b/app/cli.py @@ -19,7 +19,7 @@ from app.auth.util import random_token from app.constants import NOTIF_COMMUNITY, NOTIF_POST, NOTIF_REPLY from app.email import send_verification_email, send_email from app.models import Settings, BannedInstances, Interest, Role, User, RolePermission, Domain, ActivityPubLog, \ - utcnow, Site, Instance, File, Notification, Post, CommunityMember, NotificationSubscription, PostReply + utcnow, Site, Instance, File, Notification, Post, CommunityMember, NotificationSubscription, PostReply, Language from app.utils import file_get_contents, retrieve_block_list, blocked_domains, retrieve_peertube_block_list, \ shorten_string @@ -86,6 +86,7 @@ def register(app): db.session.add(Settings(name='allow_local_image_posts', value=json.dumps(True))) db.session.add(Settings(name='allow_remote_image_posts', value=json.dumps(True))) db.session.add(Settings(name='federation', value=json.dumps(True))) + db.session.add(Language(name='Undetermined', code='und')) banned_instances = ['anonib.al','lemmygrad.ml', 'gab.com', 'rqd2.net', 'exploding-heads.com', 'hexbear.net', 'threads.net', 'noauthority.social', 'pieville.net', 'links.hackliberty.org', 'poa.st', 'freespeechextremist.com', 'bae.st', 'nicecrew.digital', 'detroitriotcity.com', diff --git a/app/community/forms.py b/app/community/forms.py index 109ce8b9..96e1b27f 100644 --- a/app/community/forms.py +++ b/app/community/forms.py @@ -3,6 +3,7 @@ from flask_login import current_user from flask_wtf import FlaskForm from wtforms import StringField, SubmitField, TextAreaField, BooleanField, HiddenField, SelectField, FileField, \ DateField +from wtforms.fields.choices import SelectMultipleField from wtforms.validators import ValidationError, DataRequired, Email, EqualTo, Length, Regexp, Optional from flask_babel import _, lazy_gettext as _l @@ -23,6 +24,7 @@ class AddCommunityForm(FlaskForm): rules = TextAreaField(_l('Rules')) nsfw = BooleanField('NSFW') local_only = BooleanField('Local only') + languages = SelectMultipleField(_l('Languages'), coerce=int, validators=[Optional()], render_kw={'class': 'form-select'}) submit = SubmitField(_l('Create')) def validate(self, extra_validators=None): @@ -53,6 +55,7 @@ class EditCommunityForm(FlaskForm): restricted_to_mods = BooleanField(_l('Only moderators can post')) new_mods_wanted = BooleanField(_l('New moderators wanted')) topic = SelectField(_l('Topic'), coerce=int, validators=[Optional()]) + languages = SelectMultipleField(_l('Languages'), coerce=int, validators=[Optional()], render_kw={'class': 'form-select'}) layouts = [('', _l('List')), ('masonry', _l('Masonry')), ('masonry_wide', _l('Wide masonry'))] diff --git a/app/community/routes.py b/app/community/routes.py index 56481dac..21c4d2b2 100644 --- a/app/community/routes.py +++ b/app/community/routes.py @@ -25,7 +25,7 @@ from app.constants import SUBSCRIPTION_MEMBER, SUBSCRIPTION_OWNER, POST_TYPE_LIN from app.inoculation import inoculation from app.models import User, Community, CommunityMember, CommunityJoinRequest, CommunityBan, Post, \ File, PostVote, utcnow, Report, Notification, InstanceBlock, ActivityPubLog, Topic, Conversation, PostReply, \ - NotificationSubscription, UserFollower, Instance + NotificationSubscription, UserFollower, Instance, Language from app.community import bp from app.user.utils import search_for_user from app.utils import get_setting, render_template, allowlist_html, markdown_to_html, validation_required, \ @@ -33,7 +33,7 @@ from app.utils import get_setting, render_template, allowlist_html, markdown_to_ request_etag_matches, return_304, instance_banned, can_create_post, can_upvote, can_downvote, user_filters_posts, \ joined_communities, moderating_communities, blocked_domains, mimetype_from_url, blocked_instances, \ community_moderators, communities_banned_from, show_ban_message, recently_upvoted_posts, recently_downvoted_posts, \ - blocked_users, post_ranking + blocked_users, post_ranking, languages_for_form from feedgen.feed import FeedGenerator from datetime import timezone, timedelta @@ -47,6 +47,8 @@ def add_local(): if g.site.enable_nsfw is False: form.nsfw.render_kw = {'disabled': True} + form.languages.choices = languages_for_form() + if form.validate_on_submit(): if form.url.data.strip().lower().startswith('/c/'): form.url.data = form.url.data[3:] @@ -76,6 +78,11 @@ def add_local(): membership = CommunityMember(user_id=current_user.id, community_id=community.id, is_moderator=True, is_owner=True) db.session.add(membership) + # Languages of the community + for language_choice in form.languages.data: + community.languages.append(Language.query.get(language_choice)) + # Always include the undetermined language, so posts with no language will be accepted + community.languages.append(Language.query.filter(Language.code == 'und').first()) db.session.commit() flash(_('Your new community has been created.')) cache.delete_memoized(community_membership, current_user, community) @@ -940,6 +947,7 @@ def community_edit(community_id: int): if community.is_owner() or current_user.is_admin(): form = EditCommunityForm() form.topic.choices = topics_for_form(0) + form.languages.choices = languages_for_form() if form.validate_on_submit(): community.title = form.title.data community.description = form.description.data @@ -968,6 +976,14 @@ def community_edit(community_id: int): if file: community.image = file + # Languages of the community + db.session.execute(text('DELETE FROM "community_language" WHERE community_id = :community_id'), + {'community_id': community_id}) + for language_choice in form.languages.data: + community.languages.append(Language.query.get(language_choice)) + # Always include the undetermined language, so posts with no language will be accepted + community.languages.append(Language.query.filter(Language.code == 'und').first()) + db.session.commit() if community.topic: community.topic.num_communities = community.topic.communities.count() @@ -983,6 +999,7 @@ def community_edit(community_id: int): form.new_mods_wanted.data = community.new_mods_wanted form.restricted_to_mods.data = community.restricted_to_mods form.topic.data = community.topic_id if community.topic_id else None + form.languages.data = community.language_ids() form.default_layout.data = community.default_layout return render_template('community/community_edit.html', title=_('Edit community'), form=form, current_app=current_app, current="edit_settings", diff --git a/app/community/util.py b/app/community/util.py index fe8650ed..09a99a49 100644 --- a/app/community/util.py +++ b/app/community/util.py @@ -13,7 +13,7 @@ from app.activitypub.signature import post_request, default_context from app.activitypub.util import find_actor_or_create, actor_json_to_model, post_json_to_model, ensure_domains_match from app.constants import POST_TYPE_ARTICLE, POST_TYPE_LINK, POST_TYPE_IMAGE, POST_TYPE_VIDEO, NOTIF_POST from app.models import Community, File, BannedInstances, PostReply, PostVote, Post, utcnow, CommunityMember, Site, \ - Instance, Notification, User, ActivityPubLog, NotificationSubscription + Instance, Notification, User, ActivityPubLog, NotificationSubscription, Language from app.utils import get_request, gibberish, markdown_to_html, domain_from_url, allowlist_html, \ is_image_url, ensure_directory_exists, inbox_domain, post_ranking, shorten_string, parse_page, \ remove_tracking_from_link, ap_datetime, instance_banned, blocked_phrases diff --git a/app/models.py b/app/models.py index a84b5c02..1766682b 100644 --- a/app/models.py +++ b/app/models.py @@ -394,6 +394,8 @@ class Community(db.Model): show_popular = db.Column(db.Boolean, default=True) show_all = db.Column(db.Boolean, default=True) + ignore_remote_language = db.Column(db.Boolean, default=False) + search_vector = db.Column(TSVectorType('name', 'title', 'description', 'rules')) posts = db.relationship('Post', lazy='dynamic', cascade="all, delete-orphan") @@ -402,6 +404,9 @@ class Community(db.Model): image = db.relationship('File', foreign_keys=[image_id], single_parent=True, cascade="all, delete-orphan") languages = db.relationship('Language', lazy='dynamic', secondary=community_language, backref=db.backref('communities', lazy='dynamic')) + def language_ids(self): + return [language.id for language in self.languages.all()] + @cache.memoize(timeout=500) def icon_image(self, size='default') -> str: if self.icon_id is not None: diff --git a/app/templates/admin/edit_community.html b/app/templates/admin/edit_community.html index e30987b1..297a04b1 100644 --- a/app/templates/admin/edit_community.html +++ b/app/templates/admin/edit_community.html @@ -33,6 +33,7 @@ {{ render_field(form.rules) }} {{ render_field(form.nsfw) }} {{ render_field(form.restricted_to_mods) }} + {{ render_field(form.languages) }} {% if not community.is_local() %}
{{ _('Will not be overwritten by remote server') }} @@ -48,6 +49,7 @@ {{ render_field(form.topic) }} {{ render_field(form.default_layout) }} {{ render_field(form.posting_warning) }} + {{ render_field(form.ignore_remote_language) }} {% if not community.is_local() %}
{% endif %} diff --git a/app/templates/community/add_local.html b/app/templates/community/add_local.html index 34ad9a31..99609b0c 100644 --- a/app/templates/community/add_local.html +++ b/app/templates/community/add_local.html @@ -29,6 +29,7 @@ {{ render_field(form.nsfw) }} {{ render_field(form.local_only) }} {{ _('Only people using %(name)s can post or reply', name=current_app.config['SERVER_NAME']) }}. + {{ render_field(form.languages) }} {{ render_field(form.submit) }} diff --git a/app/templates/community/community_edit.html b/app/templates/community/community_edit.html index 9e3bc21b..ca4ea2c6 100644 --- a/app/templates/community/community_edit.html +++ b/app/templates/community/community_edit.html @@ -46,6 +46,7 @@ {{ render_field(form.local_only) }} {{ render_field(form.new_mods_wanted) }} {{ render_field(form.topic) }} + {{ render_field(form.languages) }} {{ render_field(form.default_layout) }}
diff --git a/app/utils.py b/app/utils.py index 046dcc69..e5b323cf 100644 --- a/app/utils.py +++ b/app/utils.py @@ -32,7 +32,7 @@ from PIL import Image from app.email import send_welcome_email from app.models import Settings, Domain, Instance, BannedInstances, User, Community, DomainBlock, ActivityPubLog, IpBan, \ - Site, Post, PostReply, utcnow, Filter, CommunityMember, InstanceBlock, CommunityBan, Topic, UserBlock + Site, Post, PostReply, utcnow, Filter, CommunityMember, InstanceBlock, CommunityBan, Topic, UserBlock, Language # Flask's render_template function, with support for themes added @@ -976,3 +976,11 @@ def recently_downvoted_post_replies(user_id) -> List[int]: reply_ids = db.session.execute(text('SELECT post_reply_id FROM "post_reply_vote" WHERE user_id = :user_id AND effect < 0 ORDER BY id DESC LIMIT 1000'), {'user_id': user_id}).scalars() return sorted(reply_ids) + + +def languages_for_form(): + result = [] + for language in Language.query.order_by(Language.name).all(): + if language.code != 'und': + result.append((language.id, language.name)) + return result diff --git a/migrations/versions/94828ddc7c63_community_remote_language.py b/migrations/versions/94828ddc7c63_community_remote_language.py new file mode 100644 index 00000000..03a46bc3 --- /dev/null +++ b/migrations/versions/94828ddc7c63_community_remote_language.py @@ -0,0 +1,32 @@ +"""community remote language + +Revision ID: 94828ddc7c63 +Revises: 5487a1886c62 +Create Date: 2024-05-08 20:55:23.821386 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '94828ddc7c63' +down_revision = '5487a1886c62' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('community', schema=None) as batch_op: + batch_op.add_column(sa.Column('ignore_remote_language', sa.Boolean(), nullable=True)) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('community', schema=None) as batch_op: + batch_op.drop_column('ignore_remote_language') + + # ### end Alembic commands ###