Merge remote-tracking branch 'upstream/main' into 25

INSTALL.md

@@ -120,9 +120,17 @@ If it does not work check the log file at logs/pyfedi.log for clues.
 <div id="initialise-database-and-setup-admin-account"></div>
 
 ## Initialise database, and set up admin account
+`export FLASK_APP=pyfedi.py`
+
+`flask db upgrade`
+
 `flask init-db`
+
 (choose a new username, email address, and password for your PyFedi admin account)
 
+If you see an error message "ModuleNotFoundError: No module named 'flask_babel'" then use `venv/bin/flask` instead of `flask`
+for all flask commands.
+
 <div id="run-the-app"></div>
 
 ## Run the app             

app/activitypub/util.py

@@ -264,6 +264,11 @@ def find_actor_or_create(actor: str, create_if_not_found=True, community_only=Fa
             return None
         if user is None:
             user = Community.query.filter(Community.ap_profile_id == actor).first()
+            if user and user.banned:
+                # Try to find a non-banned copy of the community. Sometimes duplicates happen and one copy is banned.
+                user = Community.query.filter(Community.ap_profile_id == actor).filter(Community.banned == False).first()
+                if user is None:    # no un-banned version of this community exists, only the banned one. So it was banned for being bad, not for being a duplicate.
+                    return None
 
     if user is not None:
         if not user.is_local() and (user.ap_fetched_at is None or user.ap_fetched_at < utcnow() - timedelta(days=7)):
@@ -601,7 +606,7 @@ def actor_json_to_model(activity_json, address, server):
                               ap_followers_url=activity_json['followers'],
                               ap_inbox_url=activity_json['endpoints']['sharedInbox'],
                               ap_outbox_url=activity_json['outbox'],
-                              ap_featured_url=activity_json['featured'],
+                              ap_featured_url=activity_json['featured'] if 'featured' in activity_json else '',
                               ap_moderators_url=mods_url,
                               ap_fetched_at=utcnow(),
                               ap_domain=server,
@@ -929,12 +934,15 @@ def refresh_instance_profile_task(instance_id: int):
             except requests.exceptions.JSONDecodeError as ex:
                 instance_json = {}
             if 'type' in instance_json and instance_json['type'] == 'Application':
+                # 'name' is unreliable as the admin can change it to anything. todo: find better way
                 if instance_json['name'].lower() == 'kbin':
                     software = 'Kbin'
                 elif instance_json['name'].lower() == 'mbin':
                     software = 'Mbin'
                 elif instance_json['name'].lower() == 'piefed':
                     software = 'PieFed'
+                elif instance_json['name'].lower() == 'system account':
+                    software = 'Friendica'
                 else:
                     software = 'Lemmy'
                 instance.inbox = instance_json['inbox']

app/admin/routes.py

@@ -627,7 +627,7 @@ def admin_users_add():
         private_key, public_key = RsaKeys.generate_keypair()
         user.private_key = private_key
         user.public_key = public_key
-        user.ap_profile_id = f"https://{current_app.config['SERVER_NAME']}/u/{user.user_name}"
+        user.ap_profile_id = f"https://{current_app.config['SERVER_NAME']}/u/{user.user_name}".lower()
         user.ap_public_url = f"https://{current_app.config['SERVER_NAME']}/u/{user.user_name}"
         user.ap_inbox_url = f"https://{current_app.config['SERVER_NAME']}/u/{user.user_name}/inbox"
         user.roles.append(Role.query.get(form.role.data))

app/cli.py

@@ -76,15 +76,13 @@ def register(app):
             db.configure_mappers()
             db.create_all()
             private_key, public_key = RsaKeys.generate_keypair()
-            db.session.add(Site(name="PieFed", description='', public_key=public_key, private_key=private_key))
+            db.session.add(Site(name="PieFed", description='Explore Anything, Discuss Everything.', public_key=public_key, private_key=private_key))
             db.session.add(Instance(domain=app.config['SERVER_NAME'], software='PieFed'))   # Instance 1 is always the local instance
             db.session.add(Settings(name='allow_nsfw', value=json.dumps(False)))
             db.session.add(Settings(name='allow_nsfl', value=json.dumps(False)))
             db.session.add(Settings(name='allow_dislike', value=json.dumps(True)))
             db.session.add(Settings(name='allow_local_image_posts', value=json.dumps(True)))
             db.session.add(Settings(name='allow_remote_image_posts', value=json.dumps(True)))
-            db.session.add(Settings(name='registration_open', value=json.dumps(True)))
-            db.session.add(Settings(name='approve_registrations', value=json.dumps(False)))
             db.session.add(Settings(name='federation', value=json.dumps(True)))
             banned_instances = ['anonib.al','lemmygrad.ml', 'gab.com', 'rqd2.net', 'exploding-heads.com', 'hexbear.net',
                                 'threads.net', 'noauthority.social', 'pieville.net', 'links.hackliberty.org',
@@ -95,21 +93,6 @@ def register(app):
                 db.session.add(BannedInstances(domain=bi))
                 print("Added banned instance", bi)
 
-            print("Populating DB with instances and interests")
-            print("See interests.txt")
-            interests = file_get_contents('interests.txt')
-            db.session.add(Interest(name='🕊 Chilling', communities=parse_communities(interests, 'chilling')))
-            db.session.add(Interest(name='💭 Interesting stuff', communities=parse_communities(interests, 'interesting stuff')))
-            db.session.add(Interest(name='📰 News & Politics', communities=parse_communities(interests, 'news & politics')))
-            db.session.add(Interest(name='🎮 Gaming', communities=parse_communities(interests, 'gaming')))
-            db.session.add(Interest(name='🤓 Linux', communities=parse_communities(interests, 'linux')))
-            db.session.add(Interest(name='♻️ Environment', communities=parse_communities(interests, 'environment')))
-            db.session.add(Interest(name='🏳‍🌈 LGBTQ+', communities=parse_communities(interests, 'lgbtq')))
-            db.session.add(Interest(name='🛠 Programming', communities=parse_communities(interests, 'programming')))
-            db.session.add(Interest(name='🖥️ Tech', communities=parse_communities(interests, 'tech')))
-            db.session.add(Interest(name='🤗 Mental Health', communities=parse_communities(interests, 'mental health')))
-            db.session.add(Interest(name='💊 Health', communities=parse_communities(interests, 'health')))
-
             # Load initial domain block list
             block_list = retrieve_block_list()
             if block_list:

app/community/forms.py

@@ -136,6 +136,7 @@ class CreatePostForm(FlaskForm):
                     current_user.reputation -= 1
                     db.session.commit()
                     return False
+            if self.communities:
                 community = Community.query.get(self.communities.data)
                 if community.is_local() and g.site.allow_local_image_posts is False:
                     self.communities.errors.append(_('Images cannot be posted to local communities.'))

app/community/routes.py

@@ -9,7 +9,7 @@ from sqlalchemy import or_, desc
 
 from app import db, constants, cache
 from app.activitypub.signature import RsaKeys, post_request
-from app.activitypub.util import default_context, notify_about_post, find_actor_or_create
+from app.activitypub.util import default_context, notify_about_post, find_actor_or_create, make_image_sizes
 from app.chat.util import send_message
 from app.community.forms import SearchRemoteCommunity, CreatePostForm, ReportCommunityForm, \
     DeleteCommunityForm, AddCommunityForm, EditCommunityForm, AddModeratorForm, BanUserCommunityForm
@@ -49,7 +49,8 @@ def add_local():
                               rules=form.rules.data, nsfw=form.nsfw.data, private_key=private_key,
                               public_key=public_key, description_html=markdown_to_html(form.description.data),
                               rules_html=markdown_to_html(form.rules.data), local_only=form.local_only.data,
-                              ap_profile_id='https://' + current_app.config['SERVER_NAME'] + '/c/' + form.url.data,
+                              ap_profile_id='https://' + current_app.config['SERVER_NAME'] + '/c/' + form.url.data.lower(),
+                              ap_public_url='https://' + current_app.config['SERVER_NAME'] + '/c/' + form.url.data,
                               ap_followers_url='https://' + current_app.config['SERVER_NAME'] + '/c/' + form.url.data + '/followers',
                               ap_domain=current_app.config['SERVER_NAME'],
                               subscriptions_count=1, instance_id=1, low_quality='memes' in form.url.data)
@@ -102,7 +103,7 @@ def add_remote():
                 flash(_('Community not found.'), 'warning')
             else:
                 flash(_('Community not found. If you are searching for a nsfw community it is blocked by this instance.'), 'warning')
-
+        else:
             if new_community.banned:
                 flash(_('That community is banned from %(site)s.', site=g.site.name), 'warning')
 
@@ -465,6 +466,8 @@ def add_post(actor):
         db.session.commit()
         post.ap_id = f"https://{current_app.config['SERVER_NAME']}/post/{post.id}"
         db.session.commit()
+        if post.image_id and post.image.file_path is None:
+            make_image_sizes(post.image_id, 150, 512, 'posts')  # the 512 sized image is for masonry view
 
         notify_about_post(post)
 

app/community/util.py

@@ -96,7 +96,7 @@ def retrieve_mods_and_backfill(community_id: int):
             if outbox_request.status_code == 200:
                 outbox_data = outbox_request.json()
                 outbox_request.close()
-                if outbox_data['type'] == 'OrderedCollection' and 'orderedItems' in outbox_data:
+                if 'type' in outbox_data and outbox_data['type'] == 'OrderedCollection' and 'orderedItems' in outbox_data:
                     activities_processed = 0
                     for activity in outbox_data['orderedItems']:
                         user = find_actor_or_create(activity['object']['actor'])
@@ -210,13 +210,13 @@ def save_post(form, post: Post):
                 remove_old_file(post.image_id)
                 post.image_id = None
 
-            unused, file_extension = os.path.splitext(form.link_url.data)  # do not use _ here instead of 'unused'
+            unused, file_extension = os.path.splitext(form.link_url.data)
-            # this url is a link to an image - generate a thumbnail of it
+            # this url is a link to an image - turn it into a image post
             if file_extension.lower() in allowed_extensions:
-                file = url_to_thumbnail_file(form.link_url.data)
+                file = File(source_url=form.link_url.data)
-                if file:
                 post.image = file
                 db.session.add(file)
+                post.type = POST_TYPE_IMAGE
             else:
                 # check opengraph tags on the page and make a thumbnail if an image is available in the og:image meta tag
                 opengraph = opengraph_parse(form.link_url.data)
@@ -255,6 +255,7 @@ def save_post(form, post: Post):
 
             # save the file
             final_place = os.path.join(directory, new_filename + file_ext)
+            final_place_medium = os.path.join(directory, new_filename + '_medium.webp')
             final_place_thumbnail = os.path.join(directory, new_filename + '_thumbnail.webp')
             uploaded_file.seek(0)
             uploaded_file.save(final_place)
@@ -270,18 +271,20 @@ def save_post(form, post: Post):
                 img = ImageOps.exif_transpose(img)
                 img_width = img.width
                 img_height = img.height
-                if img.width > 2000 or img.height > 2000:
                 img.thumbnail((2000, 2000))
                 img.save(final_place)
+                if img.width > 512 or img.height > 512:
+                    img.thumbnail((512, 512))
+                    img.save(final_place_medium, format="WebP", quality=93)
                     img_width = img.width
                     img_height = img.height
                 # save a second, smaller, version as a thumbnail
-                img.thumbnail((256, 256))
+                img.thumbnail((150, 150))
                 img.save(final_place_thumbnail, format="WebP", quality=93)
                 thumbnail_width = img.width
                 thumbnail_height = img.height
 
-                file = File(file_path=final_place, file_name=new_filename + file_ext, alt_text=alt_text,
+                file = File(file_path=final_place_medium, file_name=new_filename + file_ext, alt_text=alt_text,
                             width=img_width, height=img_height, thumbnail_width=thumbnail_width,
                             thumbnail_height=thumbnail_height, thumbnail_path=final_place_thumbnail,
                             source_url=final_place.replace('app/static/', f"https://{current_app.config['SERVER_NAME']}/static/"))

app/main/routes.py

@@ -343,8 +343,8 @@ def activitypub_application():
         '@context': default_context(),
         'type': 'Application',
         'id': f"https://{current_app.config['SERVER_NAME']}/",
-        'name': g.site.name,
+        'name': 'PieFed',
-        'summary': g.site.description,
+        'summary': g.site.name + ' - ' + g.site.description,
         'published': ap_datetime(g.site.created_at),
         'updated': ap_datetime(g.site.updated),
         'inbox': f"https://{current_app.config['SERVER_NAME']}/site_inbox",

app/models.py

@@ -440,6 +440,10 @@ class Community(db.Model):
         retval = self.ap_profile_id if self.ap_profile_id else f"https://{current_app.config['SERVER_NAME']}/c/{self.name}"
         return retval.lower()
 
+    def public_url(self):
+        result = self.ap_public_url if self.ap_public_url else f"https://{current_app.config['SERVER_NAME']}/c/{self.name}"
+        return result
+
     def is_local(self):
         return self.ap_id is None or self.profile_id().startswith('https://' + current_app.config['SERVER_NAME'])
 
@@ -466,6 +470,14 @@ class Community(db.Model):
         instances = instances.filter(Instance.id != 1, Instance.gone_forever == False)
         return instances.all()
 
+    def has_followers_from_domain(self, domain: str) -> bool:
+        instances = Instance.query.join(User, User.instance_id == Instance.id).join(CommunityMember, CommunityMember.user_id == User.id)
+        instances = instances.filter(CommunityMember.community_id == self.id, CommunityMember.is_banned == False)
+        for instance in instances:
+            if instance.domain == domain:
+                return True
+        return False
+
     def delete_dependencies(self):
         for post in self.posts:
             post.delete_dependencies()
@@ -750,6 +762,10 @@ class User(UserMixin, db.Model):
         result = self.ap_profile_id if self.ap_profile_id else f"https://{current_app.config['SERVER_NAME']}/u/{self.user_name}"
         return result
 
+    def public_url(self):
+        result = self.ap_public_url if self.ap_public_url else f"https://{current_app.config['SERVER_NAME']}/u/{self.user_name}"
+        return result
+
     def created_recently(self):
         return self.created and self.created > utcnow() - timedelta(days=7)
 
@@ -803,6 +819,12 @@ class User(UserMixin, db.Model):
             reply.body = reply.body_html = ''
         db.session.commit()
 
+    def mention_tag(self):
+        if self.ap_domain is None:
+            return '@' + self.user_name + '@' + current_app.config['SERVER_NAME']
+        else:
+            return '@' + self.user_name + '@' + self.ap_domain
+
 
 class ActivityLog(db.Model):
     id = db.Column(db.Integer, primary_key=True)

app/post/routes.py

@@ -118,12 +118,12 @@ def show_post(post_id: int):
         reply_json = {
             'type': 'Note',
             'id': reply.profile_id(),
-            'attributedTo': current_user.profile_id(),
+            'attributedTo': current_user.public_url(),
             'to': [
                 'https://www.w3.org/ns/activitystreams#Public'
             ],
             'cc': [
-                community.profile_id(),
+                community.public_url(), post.author.public_url()
             ],
             'content': reply.body_html,
             'inReplyTo': post.profile_id(),
@@ -134,20 +134,30 @@ def show_post(post_id: int):
             },
             'published': ap_datetime(utcnow()),
             'distinguished': False,
-            'audience': community.profile_id()
+            'audience': community.public_url(),
+            'tag': [{
+                'href': post.author.public_url(),
+                'name': post.author.mention_tag(),
+                'type': 'Mention'
+            }]
         }
         create_json = {
             'type': 'Create',
-            'actor': current_user.profile_id(),
+            'actor': current_user.public_url(),
-            'audience': community.profile_id(),
+            'audience': community.public_url(),
             'to': [
                 'https://www.w3.org/ns/activitystreams#Public'
             ],
             'cc': [
-                community.ap_profile_id
+                community.public_url(), post.author.public_url()
             ],
             'object': reply_json,
-            'id': f"https://{current_app.config['SERVER_NAME']}/activities/create/{gibberish(15)}"
+            'id': f"https://{current_app.config['SERVER_NAME']}/activities/create/{gibberish(15)}",
+            'tag': [{
+                'href': post.author.public_url(),
+                'name': post.author.mention_tag(),
+                'type': 'Mention'
+            }]
         }
         if not community.is_local():    # this is a remote community, send it to the instance that hosts it
             success = post_request(community.ap_inbox_url, create_json, current_user.private_key,
@@ -161,7 +171,7 @@ def show_post(post_id: int):
                 "to": [
                     "https://www.w3.org/ns/activitystreams#Public"
                 ],
-                "actor": community.ap_profile_id,
+                "actor": community.public_url(),
                 "cc": [
                     community.ap_followers_url
                 ],
@@ -173,6 +183,17 @@ def show_post(post_id: int):
                 if instance.inbox and not current_user.has_blocked_instance(instance.id) and not instance_banned(instance.domain):
                     send_to_remote_instance(instance.id, community.id, announce)
 
+        # send copy of Note to post author (who won't otherwise get it if no-one else on their instance is subscribed to the community)
+        if not post.author.is_local() and post.author.ap_domain != community.ap_domain:
+            if not community.is_local() or (community.is_local and not community.has_followers_from_domain(post.author.ap_domain)):
+                success = post_request(post.author.ap_inbox_url, create_json, current_user.private_key,
+                                                       current_user.ap_profile_id + '#main-key')
+                if not success:
+                    # sending to shared inbox is good enough for Mastodon, but Lemmy will reject it the local community has no followers
+                    personal_inbox = post.author.public_url() + '/inbox'
+                    post_request(personal_inbox, create_json, current_user.private_key,
+                                                       current_user.ap_profile_id + '#main-key')
+
         return redirect(url_for('activitypub.post_ap', post_id=post_id))  # redirect to current page to avoid refresh resubmitting the form
     else:
         replies = post_replies(post.id, sort)

app/static/js/markdown/downarea.js

@@ -582,7 +582,11 @@ var DownArea = (function () {
         if (self.textarea.selectionStart != self.textarea.selectionEnd) {
             end = self.textarea.value.substr(self.textarea.selectionEnd);
             var range = self.textarea.value.slice(self.textarea.selectionStart, self.textarea.selectionEnd);
-            blockquote = "".concat(blockquote).concat(range.trim());
+            var lines = range.trim().split('\n');
+            var modifiedLines = lines.map(function (line) {
+                return "> " + line.trim();
+            });
+            blockquote = modifiedLines.join('\n') + '\n';
         }
         if (start.length && start[start.length - 1] != '\n') {
             blockquote = "\n".concat(blockquote);

app/templates/base.html

@@ -246,7 +246,7 @@
         {% if post_layout == 'masonry' or post_layout == 'masonry_wide' %}
             <!-- -->
         {% endif %}
-        <script type="text/javascript" src="{{ url_for('static', filename='js/markdown/downarea.js') }}"></script>
+        <script type="text/javascript" src="{{ url_for('static', filename='js/markdown/downarea.js', changed=getmtime('js/markdown/downarea.js')) }}"></script>
     {% endif %}
     {% if theme() and file_exists('app/templates/themes/' + theme() + '/scripts.js') %}
         <script src="{{ url_for('static', filename='themes/' + theme() + '/scripts.js') }}" />

app/templates/index.html

@@ -56,7 +56,9 @@
                         </li>
                     {% endfor %}
                 </ul>
-                <p class="mt-4"><a class="btn btn-primary" href="/communities">{{ _('Explore communities') }}</a></p>
+                <p class="mt-4"><a class="btn btn-primary" href="/communities">{{ _('Explore communities') }}</a>
+                    <a class="btn btn-primary" href="/topics">{{ _('Browse topics') }}</a>
+                </p>
             </div>
         </div>
 

app/templates/list_communities.html

@@ -105,4 +105,5 @@
 {% else %}
 <p>{{ _('There are no communities yet.') }}</p>
 {% endif %}
+<p><a href="/topics" class="btn btn-primary">{{ _('Browse topics') }}</a></p>
 {% endblock %}

app/templates/search/results.html

@@ -54,7 +54,9 @@
                         </li>
                     {% endfor %}
                 </ul>
-                <p class="mt-4"><a class="btn btn-primary" href="/communities">{{ _('Explore communities') }}</a></p>
+                <p class="mt-4"><a class="btn btn-primary" href="/communities">{{ _('Explore communities') }}</a>
+                    <a class="btn btn-primary" href="/topics">{{ _('Browse topics') }}</a>
+                </p>
             </div>
         </div>
 

app/user/routes.py

@@ -34,6 +34,12 @@ def show_people():
                            joined_communities=joined_communities(current_user.get_id()), title=_('People'))
 
 
+@bp.route('/user/<int:user_id>', methods=['GET'])
+def show_profile_by_id(user_id):
+    user = User.query.get_or_404(user_id)
+    return show_profile(user)
+
+
 def show_profile(user):
     if (user.deleted or user.banned) and current_user.is_anonymous:
         abort(404)

app/utils.py

@@ -670,7 +670,7 @@ def finalize_user_setup(user, application_required=False):
     private_key, public_key = RsaKeys.generate_keypair()
     user.private_key = private_key
     user.public_key = public_key
-    user.ap_profile_id = f"https://{current_app.config['SERVER_NAME']}/u/{user.user_name}"
+    user.ap_profile_id = f"https://{current_app.config['SERVER_NAME']}/u/{user.user_name}".lower()
     user.ap_public_url = f"https://{current_app.config['SERVER_NAME']}/u/{user.user_name}"
     user.ap_inbox_url = f"https://{current_app.config['SERVER_NAME']}/u/{user.user_name}/inbox"
     db.session.commit()

requirements.txt

@@ -1,4 +1,4 @@
-urllib3==1.26.1
+urllib3==1.26.11
 Flask==2.3.2
 python-dotenv==1.0.0
 flask-wtf==1.1.1