follow remote communities - activitypub

This commit is contained in:
rimu 2023-09-08 20:04:01 +12:00
parent 5100d8ad6f
commit 20dfc5a43b
11 changed files with 326 additions and 61 deletions

View file

@ -1,6 +1,6 @@
# Contributing to PyFedi
Please discuss your ideas in an issue at https://codeberg.org/rimu/pyfedi/issues before
starting any work to ensure alignment with the roadmap, architecture and processes.
starting any large pieces of work to ensure alignment with the roadmap, architecture and processes.
Mailing list, Matrix channel, etc still to come.

View file

@ -6,11 +6,3 @@ A lemmy/kbin clone written in Python with Flask.
- Easy setup, easy to manage - few dependencies and extra software required.
- GPL.
## Contributing
Join our Maxtrix channel.
Report bugs in our issue queue.
If submitting a substantial PR, communicate your intentions to the maintainer(s) before investing your effort into coding.
This is to ensure your contribution will fit well with the rest of the code base.

View file

@ -2,12 +2,13 @@ from sqlalchemy import text
from app import db
from app.activitypub import bp
from flask import request, Response, render_template, current_app, abort, jsonify
from flask import request, Response, render_template, current_app, abort, jsonify, json
from app.activitypub.signature import HttpSignature
from app.community.routes import show_community
from app.models import User, Community
from app.models import User, Community, CommunityJoinRequest, CommunityMember, CommunityBan
from app.activitypub.util import public_key, users_total, active_half_year, active_month, local_posts, local_comments, \
post_to_activity
post_to_activity, find_actor_or_create
INBOX = []
@ -155,13 +156,7 @@ def community_profile(actor):
server = current_app.config['SERVER_NAME']
actor_data = {"@context": [
"https://www.w3.org/ns/activitystreams",
"https://w3id.org/security/v1",
{
"manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
"schema": "http://schema.org#",
"PropertyValue": "schema:PropertyValue",
"value": "schema:value"
}
"https://w3id.org/security/v1"
],
"type": "Group",
"id": f"https://{server}/c/{actor}",
@ -202,6 +197,36 @@ def community_profile(actor):
abort(404)
@bp.route('/inbox', methods=['GET', 'POST'])
def shared_inbox():
if request.method == 'POST':
request_json = request.get_json()
actor = find_actor_or_create(request_json['actor'])
if actor is not None:
if HttpSignature.verify_request(request, actor.public_key, skip_date=True):
if 'type' in request_json:
if request_json['type'] == 'Announce':
...
elif request_json['type'] == 'Follow':
# todo: send accept message if not banned
banned = CommunityBan.query.filter_by(user_id=current_user.id,
community_id=community.id).first()
...
elif request_json['type'] == 'Accept':
if request_json['object']['type'] == 'Follow':
community_ap_id = request_json['actor']
user_ap_id = request_json['object']['actor']
user = find_actor_or_create(user_ap_id)
community = find_actor_or_create(community_ap_id)
join_request = CommunityJoinRequest.query.filter_by(user_id=user.id,
community_id=community.id).first()
if join_request:
member = CommunityMember(user_id=user.id, community_id=community.id)
db.session.add(member)
db.session.commit()
@bp.route('/c/<actor>/outbox', methods=['GET'])
def community_outbox(actor):
actor = actor.strip()
@ -213,24 +238,6 @@ def community_outbox(actor):
"@context": [
"https://www.w3.org/ns/activitystreams",
"https://w3id.org/security/v1",
{
"lemmy": "https://join-lemmy.org/ns#",
"pt": "https://joinpeertube.org/ns#",
"sc": "http://schema.org/",
"commentsEnabled": "pt:commentsEnabled",
"sensitive": "as:sensitive",
"postingRestrictedToMods": "lemmy:postingRestrictedToMods",
"removeData": "lemmy:removeData",
"stickied": "lemmy:stickied",
"moderators": {
"@type": "@id",
"@id": "lemmy:moderators"
},
"expires": "as:endTime",
"distinguished": "lemmy:distinguished",
"language": "sc:inLanguage",
"identifier": "sc:identifier"
}
],
"type": "OrderedCollection",
"id": f"https://{current_app.config['SERVER_NAME']}/c/{actor}/outbox",

View file

@ -1,9 +1,12 @@
import json
import os
from datetime import datetime
from typing import Union
import markdown2
from flask import current_app
from sqlalchemy import text
from app import db
from app.models import User, Post, Community, BannedInstances
from app.models import User, Post, Community, BannedInstances, File
import time
import base64
import requests
@ -13,6 +16,8 @@ from app.constants import *
import functools
from urllib.parse import urlparse
from app.utils import get_request
def public_key():
if not os.path.exists('./public.pem'):
@ -49,7 +54,6 @@ def local_comments():
def send_activity(sender: User, host: str, content: str):
date = time.strftime('%a, %d %b %Y %H:%M:%S UTC', time.gmtime())
private_key = serialization.load_pem_private_key(sender.private_key, password=None)
@ -168,6 +172,7 @@ def validate_header_signature(body: str, host: str, date: str, signature: str) -
user = find_actor_or_create(body['actor'])
return verify_signature(user.private_key, signature, headers)
def banned_user_agents():
return [] # todo: finish this function
@ -178,13 +183,124 @@ def instance_blocked(host):
return instance is not None
def find_actor_or_create(actor):
def find_actor_or_create(actor: str) -> Union[User, Community, None]:
user = None
# actor parameter must be formatted as https://server/u/actor or https://server/c/actor
if current_app.config['SERVER_NAME'] + '/c/' in actor:
return Community.query.filter_by(name=actor).first() # finds communities formatted like https://localhost/c/*
return Community.query.filter_by(
ap_profile_id=actor).first() # finds communities formatted like https://localhost/c/*
if current_app.config['SERVER_NAME'] + '/u/' in actor:
user = User.query.filter_by(username=actor.split('/')[-1], ap_id=None).first() # finds local users
if user is None:
return None
elif actor.startswith('https://'):
server, address = extract_domain_and_actor(actor)
if instance_blocked(server):
return None
user = User.query.filter_by(ap_profile_id=actor).first() # finds users formatted like https://kbin.social/u/tables
if user is None:
# todo: retrieve user details via webfinger, etc
...
user = Community.query.filter_by(ap_profile_id=actor).first()
if user is None:
# retrieve user details via webfinger, etc
# todo: try, except block around every get_request
webfinger_data = get_request(f"https://{server}/.well-known/webfinger",
params={'resource': f"acct:{address}@{server}"})
if webfinger_data.status_code == 200:
webfinger_json = webfinger_data.json()
for links in webfinger_json['links']:
if 'rel' in links and links['rel'] == 'self': # this contains the URL of the activitypub profile
type = links['type'] if 'type' in links else 'application/activity+json'
# retrieve the activitypub profile
actor_data = get_request(links['href'], headers={'Accept': type})
# to see the structure of the json contained in actor_data, do a GET to https://lemmy.world/c/technology with header Accept: application/activity+json
if actor_data.status_code == 200:
activity_json = actor_data.json()
if activity_json['type'] == 'Person':
user = User(user_name=activity_json['preferredUsername'],
email=f"{address}@{server}",
about=parse_summary(activity_json),
created_at=activity_json['published'],
ap_id=f"{address}@{server}",
ap_public_url=activity_json['id'],
ap_profile_id=activity_json['id'],
ap_inbox_url=activity_json['endpoints']['sharedInbox'],
ap_preferred_username=activity_json['preferredUsername'],
ap_fetched_at=datetime.utcnow(),
ap_domain=server,
public_key=activity_json['publicKey']['publicKeyPem'],
# language=community_json['language'][0]['identifier'] # todo: language
)
if 'icon' in activity_json:
# todo: retrieve icon, save to disk, save more complete File record
avatar = File(source_url=activity_json['icon']['url'])
user.avatar = avatar
db.session.add(avatar)
if 'image' in activity_json:
# todo: retrieve image, save to disk, save more complete File record
cover = File(source_url=activity_json['image']['url'])
user.cover = cover
db.session.add(cover)
db.session.add(user)
db.session.commit()
return user
elif activity_json['type'] == 'Group':
community = Community(name=activity_json['preferredUsername'],
title=activity_json['name'],
description=activity_json['summary'],
nsfw=activity_json['sensitive'],
restricted_to_mods=activity_json['postingRestrictedToMods'],
created_at=activity_json['published'],
last_active=activity_json['updated'],
ap_id=f"{address[1:]}",
ap_public_url=activity_json['id'],
ap_profile_id=activity_json['id'],
ap_followers_url=activity_json['followers'],
ap_inbox_url=activity_json['endpoints']['sharedInbox'],
ap_fetched_at=datetime.utcnow(),
ap_domain=server,
public_key=activity_json['publicKey']['publicKeyPem'],
# language=community_json['language'][0]['identifier'] # todo: language
)
if 'icon' in activity_json:
# todo: retrieve icon, save to disk, save more complete File record
icon = File(source_url=activity_json['icon']['url'])
community.icon = icon
db.session.add(icon)
if 'image' in activity_json:
# todo: retrieve image, save to disk, save more complete File record
image = File(source_url=activity_json['image']['url'])
community.image = image
db.session.add(image)
db.session.add(community)
db.session.commit()
return community
return None
else:
return user
def extract_domain_and_actor(url_string: str):
# Parse the URL
parsed_url = urlparse(url_string)
# Extract the server domain name
server_domain = parsed_url.netloc
# Extract the part of the string after the last '/' character
actor = parsed_url.path.split('/')[-1]
return server_domain, actor
# create a summary from markdown if present, otherwise use html if available
def parse_summary(user_json) -> str:
if 'source' in user_json and user_json['source'].get('mediaType') == 'text/markdown':
# Convert Markdown to HTML
markdown_text = user_json['source']['content']
html_content = markdown2.markdown(markdown_text)
return html_content
elif 'summary' in user_json:
return user_json['summary']
else:
return ''

View file

@ -3,11 +3,11 @@ from flask import render_template, redirect, url_for, flash, request, make_respo
from flask_login import login_user, logout_user, current_user
from flask_babel import _
from app import db
from app.activitypub.signature import RsaKeys
from app.activitypub.signature import RsaKeys, HttpSignature
from app.community.forms import SearchRemoteCommunity, AddLocalCommunity
from app.community.util import search_for_community, community_url_exists
from app.constants import SUBSCRIPTION_MEMBER, SUBSCRIPTION_OWNER
from app.models import User, Community, CommunityMember
from app.models import User, Community, CommunityMember, CommunityJoinRequest, CommunityBan
from app.community import bp
from app.utils import get_setting
from sqlalchemy import or_
@ -25,7 +25,8 @@ def add_local():
form.url.data = form.url.data[3:]
private_key, public_key = RsaKeys.generate_keypair()
community = Community(title=form.community_name.data, name=form.url.data, description=form.description.data,
rules=form.rules.data, nsfw=form.nsfw.data, private_key=private_key, public_key=public_key,
rules=form.rules.data, nsfw=form.nsfw.data, private_key=private_key,
public_key=public_key,
subscriptions_count=1)
db.session.add(community)
db.session.commit()
@ -53,7 +54,8 @@ def add_remote():
elif '@' in address:
new_community = search_for_community('!' + address)
else:
message = Markup('Type address in the format !community@server.name. Search on <a href="https://lemmyverse.net/communities">Lemmyverse.net</a> to find some.')
message = Markup(
'Type address in the format !community@server.name. Search on <a href="https://lemmyverse.net/communities">Lemmyverse.net</a> to find some.')
flash(message, 'error')
return render_template('community/add_remote.html',
@ -85,18 +87,48 @@ def show_community(community: Community):
@bp.route('/<actor>/subscribe', methods=['GET'])
def subscribe(actor):
remote = False
actor = actor.strip()
if '@' in actor:
community = Community.query.filter_by(banned=False, ap_id=actor).first()
remote = True
else:
community = Community.query.filter_by(name=actor, banned=False, ap_id=None).first()
if community is not None:
if not current_user.subscribed(community):
membership = CommunityMember(user_id=current_user.id, community_id=community.id)
db.session.add(membership)
if remote:
# send ActivityPub message to remote community, asking to follow. Accept message will be sent to our shared inbox
join_request = CommunityJoinRequest(user_id=current_user.id, community_id=community.id)
db.session.add(join_request)
db.session.commit()
flash('You have subscribed to ' + community.title)
follow = {
"actor": f"https://{current_app.config['SERVER_NAME']}/u/{current_user.user_name}",
"to": [community.ap_id],
"object": community.ap_id,
"type": "Follow",
"id": f"https://{current_app.config['SERVER_NAME']}/activities/follow/" + join_request.id
}
try:
message = HttpSignature.signed_request(community.ap_inbox_url, follow, current_user.private_key,
current_user.ap_profile_id + '#main-key')
if message.status_code == 200:
flash('Your request to subscribe has been sent to ' + community.title)
else:
flash('Response status code was not 200', 'warning')
current_app.logger.error('Response code for subscription attempt was ' +
str(message.status_code) + ' ' + message.text)
except Exception as ex:
flash('Failed to send request to subscribe: ' + str(ex), 'error')
current_app.logger.error("Exception while trying to subscribe" + str(ex))
else: # for local communities, joining is instant
banned = CommunityBan.query.filter_by(user_id=current_user.id, community_id=community.id).first()
if banned:
flash('You cannot join this community')
member = CommunityMember(user_id=current_user.id, community_id=community.id)
db.session.add(member)
db.session.commit()
flash('You are subscribed to ' + community.title)
referrer = request.headers.get('Referer', None)
if referrer is not None:
return redirect(referrer)

View file

@ -11,3 +11,5 @@ SUBSCRIPTION_OWNER = 3
SUBSCRIPTION_MODERATOR = 2
SUBSCRIPTION_MEMBER = 1
SUBSCRIPTION_NONMEMBER = 0
SUBSCRIPTION_BANNED = -1

View file

@ -10,7 +10,8 @@ from sqlalchemy_utils.types import TSVectorType # https://sqlalchemy-searchable.
from app import db, login
import jwt
from app.constants import SUBSCRIPTION_NONMEMBER, SUBSCRIPTION_MEMBER, SUBSCRIPTION_MODERATOR, SUBSCRIPTION_OWNER
from app.constants import SUBSCRIPTION_NONMEMBER, SUBSCRIPTION_MEMBER, SUBSCRIPTION_MODERATOR, SUBSCRIPTION_OWNER, \
SUBSCRIPTION_BANNED
class File(db.Model):
@ -121,9 +122,12 @@ class User(UserMixin, db.Model):
searchable = db.Column(db.Boolean, default=True)
indexable = db.Column(db.Boolean, default=False)
ap_id = db.Column(db.String(255), index=True)
ap_profile_id = db.Column(db.String(255))
ap_public_url = db.Column(db.String(255))
avatar = db.relationship('File', foreign_keys=[avatar_id], single_parent=True, cascade="all, delete-orphan")
cover = db.relationship('File', foreign_keys=[cover_id], single_parent=True, cascade="all, delete-orphan")
ap_id = db.Column(db.String(255), index=True) # e.g. username@server
ap_profile_id = db.Column(db.String(255), index=True) # e.g. https://server/u/username
ap_public_url = db.Column(db.String(255)) # e.g. https://server/u/username
ap_fetched_at = db.Column(db.DateTime)
ap_followers_url = db.Column(db.String(255))
ap_preferred_username = db.Column(db.String(255))
@ -134,7 +138,6 @@ class User(UserMixin, db.Model):
search_vector = db.Column(TSVectorType('user_name', 'bio', 'keywords'))
activity = db.relationship('ActivityLog', backref='account', lazy='dynamic', cascade="all, delete-orphan")
avatar = db.relationship(File, foreign_keys=[avatar_id], cascade="all, delete-orphan")
posts = db.relationship('Post', backref='author', lazy='dynamic', cascade="all, delete-orphan")
post_replies = db.relationship('PostReply', backref='author', lazy='dynamic', cascade="all, delete-orphan")
@ -186,7 +189,9 @@ class User(UserMixin, db.Model):
return False
subscription:CommunityMember = CommunityMember.query.filter_by(user_id=self.id, community_id=community.id).first()
if subscription:
if subscription.is_owner:
if subscription.is_banned:
return SUBSCRIPTION_BANNED
elif subscription.is_owner:
return SUBSCRIPTION_OWNER
elif subscription.is_moderator:
return SUBSCRIPTION_MODERATOR
@ -305,6 +310,15 @@ class CommunityMember(db.Model):
created_at = db.Column(db.DateTime, default=datetime.utcnow)
class CommunityBan(db.Model):
user_id = db.Column(db.Integer, db.ForeignKey('user.id'), primary_key=True)
community_id = db.Column(db.Integer, db.ForeignKey('community.id'), primary_key=True)
banned_by = db.Column(db.Integer, db.ForeignKey('user.id'))
reason = db.Column(db.String(50))
created_at = db.Column(db.DateTime, default=datetime.utcnow)
ban_until = db.Column(db.DateTime)
class UserNote(db.Model):
id = db.Column(db.Integer, primary_key=True)
user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
@ -347,6 +361,18 @@ class Interest(db.Model):
communities = db.Column(db.Text)
class CommunityJoinRequest(db.Model):
id = db.Column(db.Integer, primary_key=True)
user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
community_id = db.Column(db.Integer, db.ForeignKey('community.id'))
class UserFollowRequest(db.Model):
id = db.Column(db.Integer, primary_key=True)
user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
follow_id = db.Column(db.Integer, db.ForeignKey('user.id'))
@login.user_loader
def load_user(id):
return User.query.get(int(id))

View file

@ -48,7 +48,7 @@
<div class="card-body">
<p>{{ community.description }}</p>
<p>{{ community.rules }}</p>
{% if len(mods) > 0 %}
{% if len(mods) > 0 and not community.private_mods %}
<h3>Moderators</h3>
<ol>
{% for mod in mods %}

View file

@ -0,0 +1,50 @@
"""follow requests
Revision ID: 251be00ae302
Revises: 01107dfe5a29
Create Date: 2023-09-08 19:07:51.474128
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '251be00ae302'
down_revision = '01107dfe5a29'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('community_join_request',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('user_id', sa.Integer(), nullable=True),
sa.Column('community_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['community_id'], ['community.id'], ),
sa.ForeignKeyConstraint(['user_id'], ['user.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_table('user_follow_request',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('user_id', sa.Integer(), nullable=True),
sa.Column('follow_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['follow_id'], ['user.id'], ),
sa.ForeignKeyConstraint(['user_id'], ['user.id'], ),
sa.PrimaryKeyConstraint('id')
)
with op.batch_alter_table('user', schema=None) as batch_op:
batch_op.create_index(batch_op.f('ix_user_ap_profile_id'), ['ap_profile_id'], unique=False)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('user', schema=None) as batch_op:
batch_op.drop_index(batch_op.f('ix_user_ap_profile_id'))
op.drop_table('user_follow_request')
op.drop_table('community_join_request')
# ### end Alembic commands ###

View file

@ -0,0 +1,39 @@
"""community ban
Revision ID: cc98a471a1ad
Revises: 251be00ae302
Create Date: 2023-09-08 20:03:14.527356
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'cc98a471a1ad'
down_revision = '251be00ae302'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('community_ban',
sa.Column('user_id', sa.Integer(), nullable=False),
sa.Column('community_id', sa.Integer(), nullable=False),
sa.Column('banned_by', sa.Integer(), nullable=True),
sa.Column('reason', sa.String(length=50), nullable=True),
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('ban_until', sa.DateTime(), nullable=True),
sa.ForeignKeyConstraint(['banned_by'], ['user.id'], ),
sa.ForeignKeyConstraint(['community_id'], ['community.id'], ),
sa.ForeignKeyConstraint(['user_id'], ['user.id'], ),
sa.PrimaryKeyConstraint('user_id', 'community_id')
)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('community_ban')
# ### end Alembic commands ###

View file

@ -19,3 +19,4 @@ pycryptodome==3.18.0
arrow==1.2.3
pyld==2.0.3
boto3==1.28.35
markdown2==2.4.10