diff --git a/app/admin/forms.py b/app/admin/forms.py index 1a1d649d..a0d6c009 100644 --- a/app/admin/forms.py +++ b/app/admin/forms.py @@ -52,6 +52,11 @@ class FederationForm(FlaskForm): submit = SubmitField(_l('Save')) +class PreLoadCommunitiesForm(FlaskForm): + communities_num = IntegerField(_l('Number of Communities to add'), default=25) + pre_load_submit = SubmitField(_l('Add Communities')) + + class EditCommunityForm(FlaskForm): title = StringField(_l('Title'), validators=[DataRequired()]) url = StringField(_l('Url'), validators=[DataRequired()]) diff --git a/app/admin/routes.py b/app/admin/routes.py index 15312b04..189ca496 100644 --- a/app/admin/routes.py +++ b/app/admin/routes.py @@ -1,6 +1,7 @@ import os from datetime import datetime, timedelta from io import BytesIO +import requests as r from time import sleep from flask import request, flash, json, url_for, current_app, redirect, g, abort @@ -13,12 +14,13 @@ from PIL import Image from app import db, celery, cache from app.activitypub.routes import process_inbox_request, process_delete_request from app.activitypub.signature import post_request, default_context -from app.activitypub.util import instance_allowed, instance_blocked +from app.activitypub.util import instance_allowed, instance_blocked, extract_domain_and_actor from app.admin.forms import FederationForm, SiteMiscForm, SiteProfileForm, EditCommunityForm, EditUserForm, \ - EditTopicForm, SendNewsletterForm, AddUserForm + EditTopicForm, SendNewsletterForm, AddUserForm, PreLoadCommunitiesForm from app.admin.util import unsubscribe_from_everything_then_delete, unsubscribe_from_community, send_newsletter, \ topics_for_form -from app.community.util import save_icon_file, save_banner_file +from app.community.util import save_icon_file, save_banner_file, search_for_community +from app.community.routes import do_subscribe from app.constants import REPORT_STATE_NEW, REPORT_STATE_ESCALATED from app.email import send_welcome_email from app.models import AllowedInstances, BannedInstances, ActivityPubLog, utcnow, Site, Community, CommunityMember, \ @@ -190,12 +192,125 @@ def admin_misc(): @permission_required('change instance settings') def admin_federation(): form = FederationForm() + preload_form = PreLoadCommunitiesForm() site = Site.query.get(1) if site is None: site = Site() # todo: finish form site.updated = utcnow() - if form.validate_on_submit(): + + # this is the pre-load communities button + if preload_form.pre_load_submit.data and preload_form.validate(): + # how many communities to add + if preload_form.communities_num.data: + communities_to_add = preload_form.communities_num.data + else: + communities_to_add = 25 + + # pull down the community.full.json + resp = r.get('https://data.lemmyverse.net/data/community.full.json') + + # asign the json from the response to a var + cfj = resp.json() + + # sort out the nsfw communities + csfw = [] + for c in cfj: + if c['nsfw']: + continue + else: + csfw.append(c) + + # sort out any that have less than 100 posts + cplt100 = [] + for c in csfw: + if c['counts']['posts'] < 100: + continue + else: + cplt100.append(c) + + # sort out any that do not have greater than 500 active users over the past week + cuawgt500 = [] + for c in cplt100: + if c['counts']['users_active_week'] < 500: + continue + else: + cuawgt500.append(c) + + # sort out any instances we have already banned + banned_instances = BannedInstances.query.all() + banned_urls = [] + cnotbanned = [] + for bi in banned_instances: + banned_urls.append(bi.domain) + for c in cuawgt500: + if c['baseurl'] in banned_urls: + continue + else: + cnotbanned.append(c) + + # sort out the 'seven things you can't say on tv' names (cursewords, ie sh*t), plus some + # "low effort" communities + # I dont know why, but some of them slip through on the first pass, so I just + # ran the list again and filter out more + # + # TO-DO: fix the need for the double filter + seven_things_plus = [ + 'shit', 'piss', 'fuck', + 'cunt', 'cocksucker', 'motherfucker', 'tits', + 'memes', 'piracy', '196', 'greentext', 'usauthoritarianism', + 'enoughmuskspam', 'political_weirdos' + ] + for c in cnotbanned: + for w in seven_things_plus: + if w in c['name']: + cnotbanned.remove(c) + for c in cnotbanned: + for w in seven_things_plus: + if w in c['name']: + cnotbanned.remove(c) + + # sort the list based on the users_active_week key + parsed_communities_sorted = sorted(cnotbanned, key=lambda c: c['counts']['users_active_week'], reverse=True) + + # get the community urls to join + community_urls_to_join = [] + + # if the admin user wants more added than we have, then just add all of them + if communities_to_add > len(parsed_communities_sorted): + communities_to_add = len(parsed_communities_sorted) + + # make the list of urls + for i in range(communities_to_add): + community_urls_to_join.append(parsed_communities_sorted[i]['url']) + + # loop through the list and send off the follow requests + # use User #1, the first instance admin + user = User.query.get(1) + pre_load_messages = [] + for c in community_urls_to_join: + # get the relevant url bits + server, community = extract_domain_and_actor(c) + # find the community + new_community = search_for_community('!' + community + '@' + server) + # subscribe to the community using alt_profile + # capture the messages returned by do_subscibe + # and show to user if instance is in debug mode + if current_app.debug: + message = do_subscribe(new_community.ap_id, user.id, main_user_name=False) + pre_load_messages.append(message) + else: + message_we_wont_do_anything_with = do_subscribe.delay(new_community.ap_id, user.id, main_user_name=False) + + if current_app.debug: + flash(_(f'Results: {pre_load_messages}')) + else: + flash(_(f'Subscription process for {communities_to_add} of {len(parsed_communities_sorted)} communities launched to background, check admin/activities for details')) + + return redirect(url_for('admin.admin_federation')) + + # this is the main settings form + elif form.validate_on_submit(): if form.use_allowlist.data: set_setting('use_allowlist', True) db.session.execute(text('DELETE FROM allowed_instances')) @@ -217,7 +332,8 @@ def admin_federation(): db.session.commit() flash(_('Admin settings saved')) - + + # this is just the regular page load elif request.method == 'GET': form.use_allowlist.data = get_setting('use_allowlist', False) form.use_blocklist.data = not form.use_allowlist.data @@ -228,7 +344,8 @@ def admin_federation(): form.blocked_phrases.data = site.blocked_phrases form.blocked_actors.data = get_setting('actor_blocked_words', '88') - return render_template('admin/federation.html', title=_('Federation settings'), form=form, + return render_template('admin/federation.html', title=_('Federation settings'), + form=form, preload_form=preload_form, current_app_debug=current_app.debug, moderating_communities=moderating_communities(current_user.get_id()), joined_communities=joined_communities(current_user.get_id()), menu_topics=menu_topics(), diff --git a/app/community/routes.py b/app/community/routes.py index 465b9880..23e98673 100644 --- a/app/community/routes.py +++ b/app/community/routes.py @@ -10,7 +10,7 @@ from flask_babel import _ from slugify import slugify from sqlalchemy import or_, desc, text -from app import db, constants, cache +from app import db, constants, cache, celery from app.activitypub.signature import RsaKeys, post_request, default_context, post_request_in_background from app.activitypub.util import notify_about_post, make_image_sizes, resolve_remote_post, extract_domain_and_actor from app.chat.util import send_message @@ -390,8 +390,21 @@ def show_community_rss(actor): @login_required @validation_required def subscribe(actor): + do_subscribe(actor, current_user.id) + referrer = request.headers.get('Referer', None) + if referrer is not None: + return redirect(referrer) + else: + return redirect('/c/' + actor) + +# this is separated out from the subscribe route so it can be used by the +# admin.admin_federation.preload_form as well +@celery.task +def do_subscribe(actor, user_id, main_user_name=True): remote = False actor = actor.strip() + user = User.query.get(user_id) + pre_load_message = {} if '@' in actor: community = Community.query.filter_by(banned=False, ap_id=actor).first() remote = True @@ -399,48 +412,73 @@ def subscribe(actor): community = Community.query.filter_by(name=actor, banned=False, ap_id=None).first() if community is not None: - if community.id in communities_banned_from(current_user.id): - abort(401) - if community_membership(current_user, community) != SUBSCRIPTION_MEMBER and community_membership(current_user, community) != SUBSCRIPTION_PENDING: - banned = CommunityBan.query.filter_by(user_id=current_user.id, community_id=community.id).first() + pre_load_message['community'] = community.ap_id + if community.id in communities_banned_from(user.id): + if main_user_name: + abort(401) + else: + pre_load_message['user_banned'] = True + if community_membership(user, community) != SUBSCRIPTION_MEMBER and community_membership(user, community) != SUBSCRIPTION_PENDING: + banned = CommunityBan.query.filter_by(user_id=user.id, community_id=community.id).first() if banned: - flash(_('You cannot join this community')) + if main_user_name: + flash(_('You cannot join this community')) + else: + pre_load_message['community_banned_by_local_instance'] = True success = True if remote: # send ActivityPub message to remote community, asking to follow. Accept message will be sent to our shared inbox - join_request = CommunityJoinRequest(user_id=current_user.id, community_id=community.id) + join_request = CommunityJoinRequest(user_id=user.id, community_id=community.id) db.session.add(join_request) db.session.commit() if community.instance.online(): follow = { - "actor": current_user.public_url(), + "actor": user.public_url(main_user_name=main_user_name), "to": [community.public_url()], "object": community.public_url(), "type": "Follow", "id": f"https://{current_app.config['SERVER_NAME']}/activities/follow/{join_request.id}" } - success = post_request(community.ap_inbox_url, follow, current_user.private_key, - current_user.public_url() + '#main-key', timeout=10) + success = post_request(community.ap_inbox_url, follow, user.private_key, + user.public_url(main_user_name=main_user_name) + '#main-key', timeout=10) if success is False or isinstance(success, str): if 'is not in allowlist' in success: - flash(_('%(name)s does not allow us to join their communities.', name=community.instance.domain), 'error') + msg_to_user = f'{community.instance.domain} does not allow us to join their communities.' + if main_user_name: + flash(_(msg_to_user), 'error') + else: + pre_load_message['status'] = msg_to_user else: - flash(_("There was a problem while trying to communicate with remote server. If other people have already joined this community it won't matter."), 'error') + msg_to_user = "There was a problem while trying to communicate with remote server. If other people have already joined this community it won't matter." + if main_user_name: + flash(_(msg_to_user), 'error') + else: + pre_load_message['status'] = msg_to_user + # for local communities, joining is instant - member = CommunityMember(user_id=current_user.id, community_id=community.id) + member = CommunityMember(user_id=user.id, community_id=community.id) db.session.add(member) db.session.commit() if success is True: - flash('You joined ' + community.title) - referrer = request.headers.get('Referer', None) - cache.delete_memoized(community_membership, current_user, community) - cache.delete_memoized(joined_communities, current_user.id) - if referrer is not None: - return redirect(referrer) + if main_user_name: + flash('You joined ' + community.title) + else: + pre_load_message['status'] = 'joined' else: - return redirect('/c/' + actor) + if not main_user_name: + pre_load_message['status'] = 'already subscribed, or subsciption pending' + + cache.delete_memoized(community_membership, user, community) + cache.delete_memoized(joined_communities, user.id) + if not main_user_name: + return pre_load_message else: - abort(404) + if main_user_name: + abort(404) + else: + pre_load_message['community'] = actor + pre_load_message['status'] = 'community not found' + return pre_load_message @bp.route('//unsubscribe', methods=['GET']) diff --git a/app/templates/admin/federation.html b/app/templates/admin/federation.html index d41cebcf..709a9fb3 100644 --- a/app/templates/admin/federation.html +++ b/app/templates/admin/federation.html @@ -15,6 +15,16 @@
+
+
+

Use this to "pre-load" known threadiverse communities, as ranked by posts and activity. The list of communities pulls from the same list as LemmyVerse. NSFW communities and communities from banned instances are excluded.

+ {% if current_app_debug %} +

*** This instance is in development mode. Loading more than 6 communities here could cause timeouts, depending on how your networking is setup. ***

+ {% endif %} + {{ render_form(preload_form) }} +
+
+
{% include 'admin/_nav.html' %}