michael/pyfedi
1
0
Fork 0
mirror of https://codeberg.org/rimu/pyfedi synced 2025-01-23 19:36:56 -08:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request 'Adding a pre-load communities function to the admin area' (#323) from JollyDevelopment/pyfedi:jollydev/community-pre-load-function into main

Browse source 
Reviewed-on: https://codeberg.org/rimu/pyfedi/pulls/323
This commit is contained in:
rimu 2024-09-14 03:20:07 +00:00
commit 77ce3020fd
4 changed files with 197 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
app/admin/forms.py
View file

@ -52,6 +52,11 @@ class FederationForm(FlaskForm):
submit = SubmitField(_l('Save'))
class PreLoadCommunitiesForm(FlaskForm):
communities_num = IntegerField(_l('Number of Communities to add'), default=25)
pre_load_submit = SubmitField(_l('Add Communities'))
class EditCommunityForm(FlaskForm):
title = StringField(_l('Title'), validators=[DataRequired()])
url = StringField(_l('Url'), validators=[DataRequired()])

129
app/admin/routes.py
View file

@ -1,6 +1,7 @@
import os
from datetime import datetime, timedelta
from io import BytesIO
import requests as r
from time import sleep
from flask import request, flash, json, url_for, current_app, redirect, g, abort
@ -13,12 +14,13 @@ from PIL import Image
from app import db, celery, cache
from app.activitypub.routes import process_inbox_request, process_delete_request
from app.activitypub.signature import post_request, default_context
from app.activitypub.util import instance_allowed, instance_blocked
from app.activitypub.util import instance_allowed, instance_blocked, extract_domain_and_actor
from app.admin.forms import FederationForm, SiteMiscForm, SiteProfileForm, EditCommunityForm, EditUserForm, \
EditTopicForm, SendNewsletterForm, AddUserForm
EditTopicForm, SendNewsletterForm, AddUserForm, PreLoadCommunitiesForm
from app.admin.util import unsubscribe_from_everything_then_delete, unsubscribe_from_community, send_newsletter, \
topics_for_form
from app.community.util import save_icon_file, save_banner_file
from app.community.util import save_icon_file, save_banner_file, search_for_community
from app.community.routes import do_subscribe
from app.constants import REPORT_STATE_NEW, REPORT_STATE_ESCALATED
from app.email import send_welcome_email
from app.models import AllowedInstances, BannedInstances, ActivityPubLog, utcnow, Site, Community, CommunityMember, \
@ -190,12 +192,125 @@ def admin_misc():
@permission_required('change instance settings')
def admin_federation():
form = FederationForm()
preload_form = PreLoadCommunitiesForm()
site = Site.query.get(1)
if site is None:
site = Site()
# todo: finish form
site.updated = utcnow()
if form.validate_on_submit():
# this is the pre-load communities button
if preload_form.pre_load_submit.data and preload_form.validate():
# how many communities to add
if preload_form.communities_num.data:
communities_to_add = preload_form.communities_num.data
else:
communities_to_add = 25
# pull down the community.full.json
resp = r.get('https://data.lemmyverse.net/data/community.full.json')
# asign the json from the response to a var
cfj = resp.json()
# sort out the nsfw communities
csfw = []
for c in cfj:
if c['nsfw']:
continue
else:
csfw.append(c)
# sort out any that have less than 100 posts
cplt100 = []
for c in csfw:
if c['counts']['posts'] < 100:
continue
else:
cplt100.append(c)
# sort out any that do not have greater than 500 active users over the past week
cuawgt500 = []
for c in cplt100:
if c['counts']['users_active_week'] < 500:
continue
else:
cuawgt500.append(c)
# sort out any instances we have already banned
banned_instances = BannedInstances.query.all()
banned_urls = []
cnotbanned = []
for bi in banned_instances:
banned_urls.append(bi.domain)
for c in cuawgt500:
if c['baseurl'] in banned_urls:
continue
else:
cnotbanned.append(c)
# sort out the 'seven things you can't say on tv' names (cursewords, ie sh*t), plus some
# "low effort" communities
# I dont know why, but some of them slip through on the first pass, so I just
# ran the list again and filter out more
#
# TO-DO: fix the need for the double filter
seven_things_plus = [
'shit', 'piss', 'fuck',
'cunt', 'cocksucker', 'motherfucker', 'tits',
'memes', 'piracy', '196', 'greentext', 'usauthoritarianism',
'enoughmuskspam', 'political_weirdos'
]
for c in cnotbanned:
for w in seven_things_plus:
if w in c['name']:
cnotbanned.remove(c)
for c in cnotbanned:
for w in seven_things_plus:
if w in c['name']:
cnotbanned.remove(c)
# sort the list based on the users_active_week key
parsed_communities_sorted = sorted(cnotbanned, key=lambda c: c['counts']['users_active_week'], reverse=True)
# get the community urls to join
community_urls_to_join = []
# if the admin user wants more added than we have, then just add all of them
if communities_to_add > len(parsed_communities_sorted):
communities_to_add = len(parsed_communities_sorted)
# make the list of urls
for i in range(communities_to_add):
community_urls_to_join.append(parsed_communities_sorted[i]['url'])
# loop through the list and send off the follow requests
# use User #1, the first instance admin
user = User.query.get(1)
pre_load_messages = []
for c in community_urls_to_join:
# get the relevant url bits
server, community = extract_domain_and_actor(c)
# find the community
new_community = search_for_community('!' + community + '@' + server)
# subscribe to the community using alt_profile
# capture the messages returned by do_subscibe
# and show to user if instance is in debug mode
if current_app.debug:
message = do_subscribe(new_community.ap_id, user.id, main_user_name=False)
pre_load_messages.append(message)
else:
message_we_wont_do_anything_with = do_subscribe.delay(new_community.ap_id, user.id, main_user_name=False)
if current_app.debug:
flash(_(f'Results: {pre_load_messages}'))
else:
flash(_(f'Subscription process for {communities_to_add} of {len(parsed_communities_sorted)} communities launched to background, check admin/activities for details'))
return redirect(url_for('admin.admin_federation'))
# this is the main settings form
elif form.validate_on_submit():
if form.use_allowlist.data:
set_setting('use_allowlist', True)
db.session.execute(text('DELETE FROM allowed_instances'))
@ -217,7 +332,8 @@ def admin_federation():
db.session.commit()
flash(_('Admin settings saved'))
# this is just the regular page load
elif request.method == 'GET':
form.use_allowlist.data = get_setting('use_allowlist', False)
form.use_blocklist.data = not form.use_allowlist.data
@ -228,7 +344,8 @@ def admin_federation():
form.blocked_phrases.data = site.blocked_phrases
form.blocked_actors.data = get_setting('actor_blocked_words', '88')
return render_template('admin/federation.html', title=_('Federation settings'), form=form,
return render_template('admin/federation.html', title=_('Federation settings'),
form=form, preload_form=preload_form, current_app_debug=current_app.debug,
moderating_communities=moderating_communities(current_user.get_id()),
joined_communities=joined_communities(current_user.get_id()),
menu_topics=menu_topics(),

80
app/community/routes.py
View file

@ -10,7 +10,7 @@ from flask_babel import _
from slugify import slugify
from sqlalchemy import or_, desc, text
from app import db, constants, cache
from app import db, constants, cache, celery
from app.activitypub.signature import RsaKeys, post_request, default_context, post_request_in_background
from app.activitypub.util import notify_about_post, make_image_sizes, resolve_remote_post, extract_domain_and_actor
from app.chat.util import send_message
@ -390,8 +390,21 @@ def show_community_rss(actor):
@login_required
@validation_required
def subscribe(actor):
do_subscribe(actor, current_user.id)
referrer = request.headers.get('Referer', None)
if referrer is not None:
return redirect(referrer)
else:
return redirect('/c/' + actor)
# this is separated out from the subscribe route so it can be used by the
# admin.admin_federation.preload_form as well
@celery.task
def do_subscribe(actor, user_id, main_user_name=True):
remote = False
actor = actor.strip()
user = User.query.get(user_id)
pre_load_message = {}
if '@' in actor:
community = Community.query.filter_by(banned=False, ap_id=actor).first()
remote = True
@ -399,48 +412,73 @@ def subscribe(actor):
community = Community.query.filter_by(name=actor, banned=False, ap_id=None).first()
if community is not None:
if community.id in communities_banned_from(current_user.id):
abort(401)
if community_membership(current_user, community) != SUBSCRIPTION_MEMBER and community_membership(current_user, community) != SUBSCRIPTION_PENDING:
banned = CommunityBan.query.filter_by(user_id=current_user.id, community_id=community.id).first()
pre_load_message['community'] = community.ap_id
if community.id in communities_banned_from(user.id):
if main_user_name:
abort(401)
else:
pre_load_message['user_banned'] = True
if community_membership(user, community) != SUBSCRIPTION_MEMBER and community_membership(user, community) != SUBSCRIPTION_PENDING:
banned = CommunityBan.query.filter_by(user_id=user.id, community_id=community.id).first()
if banned:
flash(_('You cannot join this community'))
if main_user_name:
flash(_('You cannot join this community'))
else:
pre_load_message['community_banned_by_local_instance'] = True
success = True
if remote:
# send ActivityPub message to remote community, asking to follow. Accept message will be sent to our shared inbox
join_request = CommunityJoinRequest(user_id=current_user.id, community_id=community.id)
join_request = CommunityJoinRequest(user_id=user.id, community_id=community.id)
db.session.add(join_request)
db.session.commit()
if community.instance.online():
follow = {
"actor": current_user.public_url(),
"actor": user.public_url(main_user_name=main_user_name),
"to": [community.public_url()],
"object": community.public_url(),
"type": "Follow",
"id": f"https://{current_app.config['SERVER_NAME']}/activities/follow/{join_request.id}"
}
success = post_request(community.ap_inbox_url, follow, current_user.private_key,
current_user.public_url() + '#main-key', timeout=10)
success = post_request(community.ap_inbox_url, follow, user.private_key,
user.public_url(main_user_name=main_user_name) + '#main-key', timeout=10)
if success is False or isinstance(success, str):
if 'is not in allowlist' in success:
flash(_('%(name)s does not allow us to join their communities.', name=community.instance.domain), 'error')
msg_to_user = f'{community.instance.domain} does not allow us to join their communities.'
if main_user_name:
flash(_(msg_to_user), 'error')
else:
pre_load_message['status'] = msg_to_user
else:
flash(_("There was a problem while trying to communicate with remote server. If other people have already joined this community it won't matter."), 'error')
msg_to_user = "There was a problem while trying to communicate with remote server. If other people have already joined this community it won't matter."
if main_user_name:
flash(_(msg_to_user), 'error')
else:
pre_load_message['status'] = msg_to_user
# for local communities, joining is instant
member = CommunityMember(user_id=current_user.id, community_id=community.id)
member = CommunityMember(user_id=user.id, community_id=community.id)
db.session.add(member)
db.session.commit()
if success is True:
flash('You joined ' + community.title)
referrer = request.headers.get('Referer', None)
cache.delete_memoized(community_membership, current_user, community)
cache.delete_memoized(joined_communities, current_user.id)
if referrer is not None:
return redirect(referrer)
if main_user_name:
flash('You joined ' + community.title)
else:
pre_load_message['status'] = 'joined'
else:
return redirect('/c/' + actor)
if not main_user_name:
pre_load_message['status'] = 'already subscribed, or subsciption pending'
cache.delete_memoized(community_membership, user, community)
cache.delete_memoized(joined_communities, user.id)
if not main_user_name:
return pre_load_message
else:
abort(404)
if main_user_name:
abort(404)
else:
pre_load_message['community'] = actor
pre_load_message['status'] = 'community not found'
return pre_load_message
@bp.route('/<actor>/unsubscribe', methods=['GET'])

10
app/templates/admin/federation.html
View file

@ -15,6 +15,16 @@
</div>
</div>
<hr />
<div class="row">
<div class="column">
<p>Use this to "pre-load" known threadiverse communities, as ranked by posts and activity. The list of communities pulls from the same list as <a href="https://lemmyverse.net/communities">LemmyVerse</a>. NSFW communities and communities from banned instances are excluded.</p>
{% if current_app_debug %}
<p>*** This instance is in development mode. Loading more than 6 communities here could cause timeouts, depending on how your networking is setup. ***</p>
{% endif %}
{{ render_form(preload_form) }}
</div>
</div>
<hr />
<div class="row">
<div class="col">
{% include 'admin/_nav.html' %}