Merge pull request 'Adding a pre-load communities function to the admin area' (#323) from JollyDevelopment/pyfedi:jollydev/community-pre-load-function into main

Reviewed-on: https://codeberg.org/rimu/pyfedi/pulls/323

app/admin/forms.py

@@ -52,6 +52,11 @@ class FederationForm(FlaskForm):
     submit = SubmitField(_l('Save'))
 
 
+class PreLoadCommunitiesForm(FlaskForm):
+    communities_num = IntegerField(_l('Number of Communities to add'), default=25)
+    pre_load_submit = SubmitField(_l('Add Communities'))
+
+
 class EditCommunityForm(FlaskForm):
     title = StringField(_l('Title'), validators=[DataRequired()])
     url = StringField(_l('Url'), validators=[DataRequired()])

app/admin/routes.py

@@ -1,6 +1,7 @@
 import os
 from datetime import datetime, timedelta
 from io import BytesIO
+import requests as r
 from time import sleep
 
 from flask import request, flash, json, url_for, current_app, redirect, g, abort
@@ -13,12 +14,13 @@ from PIL import Image
 from app import db, celery, cache
 from app.activitypub.routes import process_inbox_request, process_delete_request
 from app.activitypub.signature import post_request, default_context
-from app.activitypub.util import instance_allowed, instance_blocked
+from app.activitypub.util import instance_allowed, instance_blocked, extract_domain_and_actor
 from app.admin.forms import FederationForm, SiteMiscForm, SiteProfileForm, EditCommunityForm, EditUserForm, \
-    EditTopicForm, SendNewsletterForm, AddUserForm
+    EditTopicForm, SendNewsletterForm, AddUserForm, PreLoadCommunitiesForm
 from app.admin.util import unsubscribe_from_everything_then_delete, unsubscribe_from_community, send_newsletter, \
     topics_for_form
-from app.community.util import save_icon_file, save_banner_file
+from app.community.util import save_icon_file, save_banner_file, search_for_community
+from app.community.routes import do_subscribe
 from app.constants import REPORT_STATE_NEW, REPORT_STATE_ESCALATED
 from app.email import send_welcome_email
 from app.models import AllowedInstances, BannedInstances, ActivityPubLog, utcnow, Site, Community, CommunityMember, \
@@ -190,12 +192,125 @@ def admin_misc():
 @permission_required('change instance settings')
 def admin_federation():
     form = FederationForm()
+    preload_form = PreLoadCommunitiesForm()
     site = Site.query.get(1)
     if site is None:
         site = Site()
     # todo: finish form
     site.updated = utcnow()
-    if form.validate_on_submit():
+
+    # this is the pre-load communities button
+    if preload_form.pre_load_submit.data and preload_form.validate():
+        # how many communities to add
+        if preload_form.communities_num.data:
+            communities_to_add = preload_form.communities_num.data
+        else:
+            communities_to_add = 25
+
+        # pull down the community.full.json
+        resp = r.get('https://data.lemmyverse.net/data/community.full.json')
+
+        # asign the json from the response to a var
+        cfj = resp.json()
+
+        # sort out the nsfw communities
+        csfw = []
+        for c in cfj:
+            if c['nsfw']:
+                continue
+            else:
+                csfw.append(c)
+
+        # sort out any that have less than 100 posts
+        cplt100 = []
+        for c in csfw:
+            if c['counts']['posts'] < 100:
+                continue
+            else:
+                cplt100.append(c)
+
+        # sort out any that do not have greater than 500 active users over the past week
+        cuawgt500 = []
+        for c in cplt100:
+            if c['counts']['users_active_week'] < 500:
+                continue
+            else:
+                cuawgt500.append(c)
+
+        # sort out any instances we have already banned
+        banned_instances = BannedInstances.query.all()
+        banned_urls = []
+        cnotbanned = []
+        for bi in banned_instances:
+            banned_urls.append(bi.domain)
+        for c in cuawgt500:
+            if c['baseurl'] in banned_urls:
+                continue
+            else:
+                cnotbanned.append(c)
+
+        # sort out the 'seven things you can't say on tv' names (cursewords, ie sh*t), plus some
+        # "low effort" communities
+        # I dont know why, but some of them slip through on the first pass, so I just 
+        # ran the list again and filter out more
+        #
+        # TO-DO: fix the need for the double filter
+        seven_things_plus = [
+            'shit', 'piss', 'fuck', 
+            'cunt', 'cocksucker', 'motherfucker', 'tits', 
+            'memes', 'piracy', '196', 'greentext', 'usauthoritarianism',
+            'enoughmuskspam', 'political_weirdos'
+            ]
+        for c in cnotbanned:
+            for w in seven_things_plus:
+                if w in c['name']:
+                    cnotbanned.remove(c)
+        for c in cnotbanned:
+            for w in seven_things_plus:
+                if w in c['name']:
+                    cnotbanned.remove(c)
+
+        # sort the list based on the users_active_week key
+        parsed_communities_sorted = sorted(cnotbanned, key=lambda c: c['counts']['users_active_week'], reverse=True)
+
+        # get the community urls to join
+        community_urls_to_join = []
+        
+        # if the admin user wants more added than we have, then just add all of them
+        if communities_to_add > len(parsed_communities_sorted):
+            communities_to_add = len(parsed_communities_sorted) 
+        
+        # make the list of urls
+        for i in range(communities_to_add):
+            community_urls_to_join.append(parsed_communities_sorted[i]['url'])
+
+        # loop through the list and send off the follow requests
+        # use User #1, the first instance admin
+        user = User.query.get(1)
+        pre_load_messages = []
+        for c in community_urls_to_join:
+            # get the relevant url bits 
+            server, community = extract_domain_and_actor(c)
+            # find the community
+            new_community = search_for_community('!' + community + '@' + server)
+            # subscribe to the community using alt_profile
+            # capture the messages returned by do_subscibe
+            # and show to user if instance is in debug mode
+            if current_app.debug:
+                message = do_subscribe(new_community.ap_id, user.id, main_user_name=False)
+                pre_load_messages.append(message)
+            else:
+                message_we_wont_do_anything_with = do_subscribe.delay(new_community.ap_id, user.id, main_user_name=False)
+
+        if current_app.debug:
+            flash(_(f'Results: {pre_load_messages}'))
+        else:
+            flash(_(f'Subscription process for {communities_to_add} of {len(parsed_communities_sorted)} communities launched to background, check admin/activities for details'))
+
+        return redirect(url_for('admin.admin_federation'))
+    
+    # this is the main settings form
+    elif form.validate_on_submit():
         if form.use_allowlist.data:
             set_setting('use_allowlist', True)
             db.session.execute(text('DELETE FROM allowed_instances'))
@@ -217,7 +332,8 @@ def admin_federation():
         db.session.commit()
 
         flash(_('Admin settings saved'))
-
+    
+    # this is just the regular page load
     elif request.method == 'GET':
         form.use_allowlist.data = get_setting('use_allowlist', False)
         form.use_blocklist.data = not form.use_allowlist.data
@@ -228,7 +344,8 @@ def admin_federation():
         form.blocked_phrases.data = site.blocked_phrases
         form.blocked_actors.data = get_setting('actor_blocked_words', '88')
 
-    return render_template('admin/federation.html', title=_('Federation settings'), form=form,
+    return render_template('admin/federation.html', title=_('Federation settings'), 
+                           form=form, preload_form=preload_form, current_app_debug=current_app.debug,
                            moderating_communities=moderating_communities(current_user.get_id()),
                            joined_communities=joined_communities(current_user.get_id()),
                            menu_topics=menu_topics(),

app/community/routes.py

@@ -10,7 +10,7 @@ from flask_babel import _
 from slugify import slugify
 from sqlalchemy import or_, desc, text
 
-from app import db, constants, cache
+from app import db, constants, cache, celery
 from app.activitypub.signature import RsaKeys, post_request, default_context, post_request_in_background
 from app.activitypub.util import notify_about_post, make_image_sizes, resolve_remote_post, extract_domain_and_actor
 from app.chat.util import send_message
@@ -390,8 +390,21 @@ def show_community_rss(actor):
 @login_required
 @validation_required
 def subscribe(actor):
+    do_subscribe(actor, current_user.id)
+    referrer = request.headers.get('Referer', None)
+    if referrer is not None:
+        return redirect(referrer)
+    else:
+        return redirect('/c/' + actor)
+
+# this is separated out from the subscribe route so it can be used by the 
+# admin.admin_federation.preload_form as well
+@celery.task
+def do_subscribe(actor, user_id, main_user_name=True):
     remote = False
     actor = actor.strip()
+    user = User.query.get(user_id)
+    pre_load_message = {}
     if '@' in actor:
         community = Community.query.filter_by(banned=False, ap_id=actor).first()
         remote = True
@@ -399,48 +412,73 @@ def subscribe(actor):
         community = Community.query.filter_by(name=actor, banned=False, ap_id=None).first()
 
     if community is not None:
-        if community.id in communities_banned_from(current_user.id):
+        pre_load_message['community'] = community.ap_id
-            abort(401)
+        if community.id in communities_banned_from(user.id):
-        if community_membership(current_user, community) != SUBSCRIPTION_MEMBER and community_membership(current_user, community) != SUBSCRIPTION_PENDING:
+            if main_user_name:
-            banned = CommunityBan.query.filter_by(user_id=current_user.id, community_id=community.id).first()
+                abort(401)
+            else:
+                pre_load_message['user_banned'] = True
+        if community_membership(user, community) != SUBSCRIPTION_MEMBER and community_membership(user, community) != SUBSCRIPTION_PENDING:
+            banned = CommunityBan.query.filter_by(user_id=user.id, community_id=community.id).first()
             if banned:
-                flash(_('You cannot join this community'))
+                if main_user_name:
+                    flash(_('You cannot join this community'))
+                else:
+                    pre_load_message['community_banned_by_local_instance'] = True
             success = True
             if remote:
                 # send ActivityPub message to remote community, asking to follow. Accept message will be sent to our shared inbox
-                join_request = CommunityJoinRequest(user_id=current_user.id, community_id=community.id)
+                join_request = CommunityJoinRequest(user_id=user.id, community_id=community.id)
                 db.session.add(join_request)
                 db.session.commit()
                 if community.instance.online():
                     follow = {
-                      "actor": current_user.public_url(),
+                      "actor": user.public_url(main_user_name=main_user_name),
                       "to": [community.public_url()],
                       "object": community.public_url(),
                       "type": "Follow",
                       "id": f"https://{current_app.config['SERVER_NAME']}/activities/follow/{join_request.id}"
                     }
-                    success = post_request(community.ap_inbox_url, follow, current_user.private_key,
+                    success = post_request(community.ap_inbox_url, follow, user.private_key,
-                                                           current_user.public_url() + '#main-key', timeout=10)
+                                                           user.public_url(main_user_name=main_user_name) + '#main-key', timeout=10)
                 if success is False or isinstance(success, str):
                     if 'is not in allowlist' in success:
-                        flash(_('%(name)s does not allow us to join their communities.', name=community.instance.domain), 'error')
+                        msg_to_user = f'{community.instance.domain} does not allow us to join their communities.'
+                        if main_user_name:
+                            flash(_(msg_to_user), 'error')
+                        else:
+                            pre_load_message['status'] = msg_to_user
                     else:
-                        flash(_("There was a problem while trying to communicate with remote server. If other people have already joined this community it won't matter."), 'error')
+                        msg_to_user = "There was a problem while trying to communicate with remote server. If other people have already joined this community it won't matter."
+                        if main_user_name:
+                            flash(_(msg_to_user), 'error')
+                        else:
+                            pre_load_message['status'] = msg_to_user
+
             # for local communities, joining is instant
-            member = CommunityMember(user_id=current_user.id, community_id=community.id)
+            member = CommunityMember(user_id=user.id, community_id=community.id)
             db.session.add(member)
             db.session.commit()
             if success is True:
-                flash('You joined ' + community.title)
+                if main_user_name:
-        referrer = request.headers.get('Referer', None)
+                    flash('You joined ' + community.title)
-        cache.delete_memoized(community_membership, current_user, community)
+                else:
-        cache.delete_memoized(joined_communities, current_user.id)
+                    pre_load_message['status'] = 'joined'
-        if referrer is not None:
-            return redirect(referrer)
         else:
-            return redirect('/c/' + actor)
+            if not main_user_name:
+                pre_load_message['status'] = 'already subscribed, or subsciption pending'
+        
+        cache.delete_memoized(community_membership, user, community)
+        cache.delete_memoized(joined_communities, user.id)
+        if not main_user_name:
+            return pre_load_message
     else:
-        abort(404)
+        if main_user_name:
+            abort(404)
+        else:
+            pre_load_message['community'] = actor
+            pre_load_message['status'] = 'community not found'
+            return pre_load_message
 
 
 @bp.route('/<actor>/unsubscribe', methods=['GET'])

app/templates/admin/federation.html

@@ -15,6 +15,16 @@
     </div>
 </div>
 <hr />
+<div class="row">
+    <div class="column">
+        <p>Use this to "pre-load" known threadiverse communities, as ranked by posts and activity. The list of communities pulls from the same list as <a href="https://lemmyverse.net/communities">LemmyVerse</a>. NSFW communities and communities from banned instances are excluded.</p>
+        {% if current_app_debug %}
+        <p>*** This instance is in development mode. Loading more than 6 communities here could cause timeouts, depending on how your networking is setup. ***</p>
+        {% endif %}
+        {{ render_form(preload_form) }}
+    </div>
+</div>
+<hr />
 <div class="row">
     <div class="col">
         {% include 'admin/_nav.html' %}