Merge remote-tracking branch 'upstream/main'

app/activitypub/routes.py

@@ -195,7 +195,7 @@ def user_profile(actor):
         if is_activitypub_request():
             server = current_app.config['SERVER_NAME']
             actor_data = {  "@context": default_context(),
-                            "type": "Person",
+                            "type": "Person" if not user.bot else "Service",
                             "id": f"https://{server}/u/{actor}",
                             "preferredUsername": actor,
                             "name": user.title if user.title else user.user_name,

app/activitypub/util.py

@@ -488,7 +488,7 @@ def refresh_community_profile_task(community_id):
 
 
 def actor_json_to_model(activity_json, address, server):
-    if activity_json['type'] == 'Person':
+    if activity_json['type'] == 'Person' or activity_json['type'] == 'Service':
         try:
             user = User(user_name=activity_json['preferredUsername'],
                         title=activity_json['name'] if 'name' in activity_json else None,
@@ -508,6 +508,7 @@ def actor_json_to_model(activity_json, address, server):
                         ap_fetched_at=utcnow(),
                         ap_domain=server,
                         public_key=activity_json['publicKey']['publicKeyPem'],
+                        bot=True if activity_json['type'] == 'Service' else False,
                         instance_id=find_instance_id(server)
                         # language=community_json['language'][0]['identifier'] # todo: language
                         )
@@ -1158,6 +1159,7 @@ def create_post_reply(activity_log: ActivityPubLog, community: Community, in_rep
                                root_id=root_id,
                                nsfw=community.nsfw,
                                nsfl=community.nsfl,
+                               from_bot=user.bot,
                                up_votes=1,
                                depth=depth,
                                score=instance_weight(user.ap_domain),
@@ -1256,6 +1258,7 @@ def create_post(activity_log: ActivityPubLog, community: Community, request_json
                 ap_announce_id=announce_id,
                 type=constants.POST_TYPE_ARTICLE,
                 up_votes=1,
+                from_bot=user.bot,
                 score=instance_weight(user.ap_domain),
                 instance_id=user.instance_id,
                 indexable=user.indexable

app/community/forms.py

@@ -68,7 +68,7 @@ class SearchRemoteCommunity(FlaskForm):
 
 class BanUserCommunityForm(FlaskForm):
     reason = StringField(_l('Reason'), render_kw={'autofocus': True}, validators=[DataRequired()])
-    ban_until = DateField(_l('Ban until'))
+    ban_until = DateField(_l('Ban until'), validators=[Optional()])
     delete_posts = BooleanField(_l('Also delete all their posts'))
     delete_post_replies = BooleanField(_l('Also delete all their comments'))
     submit = SubmitField(_l('Ban'))

app/community/routes.py

@@ -27,7 +27,7 @@ from app.utils import get_setting, render_template, allowlist_html, markdown_to_
     shorten_string, gibberish, community_membership, ap_datetime, \
     request_etag_matches, return_304, instance_banned, can_create_post, can_upvote, can_downvote, user_filters_posts, \
     joined_communities, moderating_communities, blocked_domains, mimetype_from_url, blocked_instances, \
-    community_moderators
+    community_moderators, communities_banned_from
 from feedgen.feed import FeedGenerator
 from datetime import timezone, timedelta
 
@@ -793,6 +793,9 @@ def community_ban_user(community_id: int, user_id: int):
 
     form = BanUserCommunityForm()
     if form.validate_on_submit():
+        # Both CommunityBan and CommunityMember need to be updated. CommunityBan is under the control of moderators while
+        # CommunityMember can be cleared by the user by leaving the group and rejoining. CommunityMember.is_banned stops
+        # posts from the community from showing up in the banned person's home feed.
         if not existing:
             new_ban = CommunityBan(community_id=community_id, user_id=user.id, banned_by=current_user.id,
                                    reason=form.reason.data)
@@ -800,6 +803,12 @@ def community_ban_user(community_id: int, user_id: int):
                 new_ban.ban_until = form.ban_until.data
             db.session.add(new_ban)
             db.session.commit()
+
+        community_membership_record = CommunityMember.query.filter_by(community_id=community.id, user_id=user.id).first()
+        if community_membership_record:
+            community_membership_record.is_banned = True
+            db.session.commit()
+
         flash(_('%(name)s has been banned.', name=user.display_name()))
 
         if form.delete_posts.data:
@@ -819,8 +828,11 @@ def community_ban_user(community_id: int, user_id: int):
 
         # notify banned person
         if user.is_local():
+            cache.delete_memoized(communities_banned_from, user.id)
+            cache.delete_memoized(joined_communities, user.id)
+            cache.delete_memoized(moderating_communities, user.id)
             notify = Notification(title=shorten_string('You have been banned from ' + community.title),
-                                  url=f'/', user_id=user.id,
+                                  url=f'/notifications', user_id=user.id,
                                   author_id=1)
             db.session.add(notify)
             user.unread_notifications += 1
@@ -839,6 +851,42 @@ def community_ban_user(community_id: int, user_id: int):
                                )
 
 
+@bp.route('/community/<int:community_id>/<int:user_id>/unban_user_community', methods=['GET', 'POST'])
+@login_required
+def community_unban_user(community_id: int, user_id: int):
+    community = Community.query.get_or_404(community_id)
+    user = User.query.get_or_404(user_id)
+    existing_ban = CommunityBan.query.filter_by(community_id=community.id, user_id=user.id).first()
+    if existing_ban:
+        db.session.delete(existing_ban)
+        db.session.commit()
+
+    community_membership_record = CommunityMember.query.filter_by(community_id=community.id, user_id=user.id).first()
+    if community_membership_record:
+        community_membership_record.is_banned = False
+        db.session.commit()
+
+    flash(_('%(name)s has been unbanned.', name=user.display_name()))
+
+    # todo: federate ban to post author instance
+
+    # notify banned person
+    if user.is_local():
+        cache.delete_memoized(communities_banned_from, user.id)
+        cache.delete_memoized(joined_communities, user.id)
+        cache.delete_memoized(moderating_communities, user.id)
+        notify = Notification(title=shorten_string('You have been un-banned from ' + community.title),
+                              url=f'/notifications', user_id=user.id,
+                              author_id=1)
+        db.session.add(notify)
+        user.unread_notifications += 1
+        db.session.commit()
+    else:
+        ...
+        # todo: send chatmessage to remote user and federate it
+
+    return redirect(url_for('community.community_moderate_banned', actor=community.link()))
+
 
 @bp.route('/<int:community_id>/notification', methods=['GET', 'POST'])
 @login_required

app/main/routes.py

@@ -24,7 +24,7 @@ from sqlalchemy_searchable import search
 from app.utils import render_template, get_setting, gibberish, request_etag_matches, return_304, blocked_domains, \
     ap_datetime, ip_address, retrieve_block_list, shorten_string, markdown_to_text, user_filters_home, \
     joined_communities, moderating_communities, parse_page, theme_list, get_request, markdown_to_html, allowlist_html, \
-    blocked_instances
+    blocked_instances, communities_banned_from
 from app.models import Community, CommunityMember, Post, Site, User, utcnow, Domain, Topic, File, Instance, \
     InstanceRole, Notification
 from PIL import Image
@@ -131,7 +131,13 @@ def home_page(type, sort):
         next_url = url_for('main.all_posts', page=posts.next_num, sort=sort) if posts.has_next else None
         prev_url = url_for('main.all_posts', page=posts.prev_num, sort=sort) if posts.has_prev and page != 1 else None
 
-    active_communities = Community.query.filter_by(banned=False).order_by(desc(Community.last_active)).limit(5).all()
+    # Active Communities
+    active_communities = Community.query.filter_by(banned=False)
+    if current_user.is_authenticated:   # do not show communities current user is banned from
+        banned_from = communities_banned_from(current_user.id)
+        if banned_from:
+            active_communities = active_communities.filter(Community.id.not_in(banned_from))
+    active_communities = active_communities.order_by(desc(Community.last_active)).limit(5).all()
 
     return render_template('index.html', posts=posts, active_communities=active_communities, show_post_community=True,
                            POST_TYPE_IMAGE=POST_TYPE_IMAGE, POST_TYPE_LINK=POST_TYPE_LINK,
@@ -178,6 +184,11 @@ def list_communities():
     if topic_id != 0:
         communities = communities.filter_by(topic_id=topic_id)
 
+    if current_user.is_authenticated:
+        banned_from = communities_banned_from(current_user.id)
+        if banned_from:
+            communities = communities.filter(Community.id.not_in(banned_from))
+
     return render_template('list_communities.html', communities=communities.order_by(sort_by).all(), search=search_param, title=_('Communities'),
                            SUBSCRIPTION_PENDING=SUBSCRIPTION_PENDING, SUBSCRIPTION_MEMBER=SUBSCRIPTION_MEMBER,
                            SUBSCRIPTION_OWNER=SUBSCRIPTION_OWNER, SUBSCRIPTION_MODERATOR=SUBSCRIPTION_MODERATOR,

app/models.py

@@ -579,8 +579,8 @@ class User(UserMixin, db.Model):
 
     def num_content(self):
         content = 0
-        content += db.session.execute(text('SELECT COUNT(id) as c FROM "post" WHERE user_id = ' + str(self.id))).scalar()
-        content += db.session.execute(text('SELECT COUNT(id) as c FROM "post_reply" WHERE user_id = ' + str(self.id))).scalar()
+        content += db.session.execute(text('SELECT COUNT(id) as c FROM "post" WHERE user_id = :user_id'), {'user_id': self.id}).scalar()
+        content += db.session.execute(text('SELECT COUNT(id) as c FROM "post_reply" WHERE user_id = :user_id'), {'user_id': self.id}).scalar()
         return content
 
     def is_local(self):

app/post/util.py

@@ -15,6 +15,8 @@ def post_replies(post_id: int, sort_by: str, show_first: int = 0) -> List[PostRe
         instance_ids = blocked_instances(current_user.id)
         if instance_ids:
             comments = comments.filter(or_(PostReply.instance_id.not_in(instance_ids), PostReply.instance_id == None))
+        if current_user.ignore_bots:
+            comments = comments.filter(PostReply.from_bot == False)
     if sort_by == 'hot':
         comments = comments.order_by(desc(PostReply.ranking))
     elif sort_by == 'top':

app/search/routes.py

@@ -5,7 +5,8 @@ from sqlalchemy import or_
 
 from app.models import Post
 from app.search import bp
-from app.utils import moderating_communities, joined_communities, render_template, blocked_domains, blocked_instances
+from app.utils import moderating_communities, joined_communities, render_template, blocked_domains, blocked_instances, \
+    communities_banned_from
 
 
 @bp.route('/search', methods=['GET', 'POST'])
@@ -29,6 +30,9 @@ def run_search():
             instance_ids = blocked_instances(current_user.id)
             if instance_ids:
                 posts = posts.filter(or_(Post.instance_id.not_in(instance_ids), Post.instance_id == None))
+            banned_from = communities_banned_from(current_user.id)
+            if banned_from:
+                posts = posts.filter(Post.community_id.not_in(banned_from))
         else:
             posts = posts.filter(Post.from_bot == False)
             posts = posts.filter(Post.nsfl == False)
@@ -43,7 +47,7 @@ def run_search():
         prev_url = url_for('search.run_search', page=posts.prev_num, q=q) if posts.has_prev and page != 1 else None
 
         return render_template('search/results.html', title=_('Search results for %(q)s', q=q), posts=posts, q=q,
-                               next_url=next_url, prev_url=prev_url,
+                               next_url=next_url, prev_url=prev_url, show_post_community=True,
                                moderating_communities=moderating_communities(current_user.get_id()),
                                joined_communities=joined_communities(current_user.get_id()),
                                site=g.site)

app/static/scss/_typography.scss

@@ -282,6 +282,14 @@ h1 {
   }
 }
 
+.fe-bot-account {
+  position: relative;
+  top: 1px;
+  &:before {
+    content: "\e94d";
+  }
+}
+
 .fe-video {
   position: relative;
   top: 2px;

app/static/structure.css

@@ -305,6 +305,14 @@ h1 .fe-bell, h1 .fe-no-bell {
   content: "\e986";
 }
 
+.fe-bot-account {
+  position: relative;
+  top: 1px;
+}
+.fe-bot-account:before {
+  content: "\e94d";
+}
+
 .fe-video {
   position: relative;
   top: 2px;

app/static/styles.css

@@ -304,6 +304,14 @@ h1 .fe-bell, h1 .fe-no-bell {
   content: "\e986";
 }
 
+.fe-bot-account {
+  position: relative;
+  top: 1px;
+}
+.fe-bot-account:before {
+  content: "\e94d";
+}
+
 .fe-video {
   position: relative;
   top: 2px;

app/templates/base.html

@@ -11,6 +11,9 @@
         {% if user.created_recently() %}
             <span class="fe fe-new-account" title="New account"> </span>
         {% endif %}
+        {% if user.bot %}
+            <span class="fe fe-bot-account" title="Bot account"> </span>
+        {% endif %}
         {% if user.reputation < -10 %}
             <span class="fe fe-warning red" title="Very low reputation. Beware."> </span>
             <span class="fe fe-warning red" title="Very low reputation. Beware!"> </span>

app/templates/community/community_moderate_banned.html

@@ -48,11 +48,12 @@
                         <td>{{ user.reports if user.reports > 0 }} </td>
                         <td>{{ user.ip_address if user.ip_address }} </td>
                         <td>{% if user.is_local() %}
-                                <a href="/u/{{ user.link() }}">View local</a>
+                                <a href="/u/{{ user.link() }}">View</a>
                             {% else %}
+                                <a href="/u/{{ user.link() }}">View local</a> |
                                 <a href="{{ user.ap_profile_id }}">View remote</a>
                             {% endif %}
-                            | <a href="#" class="confirm_first">{{ _('Un ban') }}</a>
+                            | <a href="{{ url_for('community.community_unban_user', community_id=community.id, user_id=user.id) }}" class="confirm_first">{{ _('Un ban') }}</a>
                         </td>
                     </tr>
                 {% endfor %}

app/templates/post/continue_discussion.html

@@ -29,6 +29,9 @@
                                 {% if comment['comment'].author.created_recently() %}
                                     <span class="fe fe-new-account small" title="New account"> </span>
                                 {% endif %}
+                                {% if comment['comment'].author.bot %}
+                                    <span class="fe fe-bot-account small" title="Bot account"> </span>
+                                {% endif %}
                                 {% if comment['comment'].author.id != current_user.id %}
                                     {% if comment['comment'].author.reputation < -10 %}
                                         <span class="fe fe-warning red" title="Very low reputation. Beware."> </span>

app/templates/post/post.html

@@ -90,6 +90,9 @@
                                 {% if comment['comment'].author.created_recently() %}
                                     <span class="fe fe-new-account small" title="New account"> </span>
                                 {% endif %}
+                                {% if comment['comment'].author.bot %}
+                                    <span class="fe fe-bot-account small" title="Bot account"> </span>
+                                {% endif %}
                                 {% if comment['comment'].author.id != current_user.id %}
                                     {% if comment['comment'].author.reputation < -10 %}
                                         <span class="fe fe-warning red" title="Very low reputation. Beware."> </span>

app/templates/user/show_profile.html

@@ -57,6 +57,9 @@
             </div>
         {% endif %}
         <p class="small">{{ _('Joined') }}: {{ moment(user.created).fromNow(refresh=True) }}<br />
+            {% if user.bot %}
+                {{ _('Bot Account') }}<br />
+            {% endif %}
             {{ _('Attitude') }}: <span title="{{ _('Ratio of upvotes cast to downvotes cast. Higher is more positive.') }}">{{ (user.attitude * 100) | round | int }}%</span></p>
         {{ user.about_html|safe }}
         {% if posts %}

app/topic/routes.py

@@ -16,7 +16,8 @@ from app.topic import bp
 from app import db, celery, cache
 from app.topic.forms import ChooseTopicsForm
 from app.utils import render_template, user_filters_posts, moderating_communities, joined_communities, \
-    community_membership, blocked_domains, validation_required, mimetype_from_url, blocked_instances
+    community_membership, blocked_domains, validation_required, mimetype_from_url, blocked_instances, \
+    communities_banned_from
 
 
 @bp.route('/topic/<path:topic_path>', methods=['GET'])
@@ -68,6 +69,9 @@ def show_topic(topic_path):
             instance_ids = blocked_instances(current_user.id)
             if instance_ids:
                 posts = posts.filter(or_(Post.instance_id.not_in(instance_ids), Post.instance_id == None))
+            banned_from = communities_banned_from(current_user.id)
+            if banned_from:
+                posts = posts.filter(Post.community_id.not_in(banned_from))
 
         # sorting
         if sort == '' or sort == 'hot':

app/user/routes.py

@@ -79,7 +79,7 @@ def show_profile(user):
                            description=description, subscribed=subscribed, upvoted=upvoted,
                            post_next_url=post_next_url, post_prev_url=post_prev_url,
                            replies_next_url=replies_next_url, replies_prev_url=replies_prev_url,
-                           noindex=not user.indexable,
+                           noindex=not user.indexable, show_post_community=True,
                            moderating_communities=moderating_communities(current_user.get_id()),
                            joined_communities=joined_communities(current_user.get_id())
                            )

app/user/utils.py

@@ -149,7 +149,7 @@ def search_for_user(address: str):
                 if user_data.status_code == 200:
                     user_json = user_data.json()
                     user_data.close()
-                    if user_json['type'] == 'Person':
+                    if user_json['type'] == 'Person' or user_json['type'] == 'Service':
                         user = actor_json_to_model(user_json, name, server)
                         return user
     return None

app/utils.py

@@ -28,7 +28,7 @@ import re
 
 from app.email import send_welcome_email
 from app.models import Settings, Domain, Instance, BannedInstances, User, Community, DomainBlock, ActivityPubLog, IpBan, \
-    Site, Post, PostReply, utcnow, Filter, CommunityMember, InstanceBlock
+    Site, Post, PostReply, utcnow, Filter, CommunityMember, InstanceBlock, CommunityBan
 
 
 # Flask's render_template function, with support for themes added
@@ -251,7 +251,7 @@ def html_to_markdown_worker(element, indent_level=0):
 
 def markdown_to_html(markdown_text) -> str:
     if markdown_text:
-        return allowlist_html(markdown2.markdown(markdown_text, safe_mode=True, extras={'middle-word-em': False, 'tables': True, 'fenced-code-blocks': True, 'spoiler': True}))
+        return allowlist_html(markdown2.markdown(markdown_text, safe_mode=True, extras={'middle-word-em': False, 'tables': True, 'fenced-code-blocks': True, 'strike': True}))
     else:
         return ''
 
@@ -312,6 +312,12 @@ def community_membership(user: User, community: Community) -> int:
     return user.subscribed(community.id)
 
 
+@cache.memoize(timeout=86400)
+def communities_banned_from(user_id) -> List[int]:
+    community_bans = CommunityBan.query.filter(CommunityBan.user_id == user_id).all()
+    return [cb.community_id for cb in community_bans]
+
+
 @cache.memoize(timeout=86400)
 def blocked_domains(user_id) -> List[int]:
     blocks = DomainBlock.query.filter_by(user_id=user_id)
@@ -442,7 +448,7 @@ def banned_ip_addresses() -> List[str]:
 
 
 def can_downvote(user, community: Community, site=None) -> bool:
-    if user is None or community is None or user.banned:
+    if user is None or community is None or user.banned or user.bot:
         return False
 
     if site is None:
@@ -460,11 +466,17 @@ def can_downvote(user, community: Community, site=None) -> bool:
     if user.attitude < -0.40 or user.reputation < -10:  # this should exclude about 3.7% of users.
         return False
 
+    if community.id in communities_banned_from(user.id):
+        return False
+
     return True
 
 
 def can_upvote(user, community: Community) -> bool:
-    if user is None or community is None or user.banned:
+    if user is None or community is None or user.banned or user.bot:
+        return False
+
+    if community.id in communities_banned_from(user.id):
         return False
 
     return True
@@ -483,6 +495,9 @@ def can_create_post(user, content: Community) -> bool:
     if content.local_only and not user.is_local():
         return False
 
+    if content.id in communities_banned_from(user.id):
+        return False
+
     return True
 
 
@@ -496,6 +511,9 @@ def can_create_post_reply(user, content: Community) -> bool:
     if content.local_only and not user.is_local():
         return False
 
+    if content.id in communities_banned_from(user.id):
+        return False
+
     return True
 
 
@@ -603,6 +621,7 @@ def moderating_communities(user_id):
     return Community.query.join(CommunityMember, Community.id == CommunityMember.community_id).\
         filter(Community.banned == False).\
         filter(or_(CommunityMember.is_moderator == True, CommunityMember.is_owner == True)). \
+        filter(CommunityMember.is_banned == False). \
         filter(CommunityMember.user_id == user_id).order_by(Community.title).all()
 
 
@@ -613,6 +632,7 @@ def joined_communities(user_id):
     return Community.query.join(CommunityMember, Community.id == CommunityMember.community_id).\
         filter(Community.banned == False). \
         filter(CommunityMember.is_moderator == False, CommunityMember.is_owner == False). \
+        filter(CommunityMember.is_banned == False). \
         filter(CommunityMember.user_id == user_id).order_by(Community.title).all()
 
 

config.py

@@ -42,3 +42,7 @@ class Config(object):
     SENTRY_DSN = os.environ.get('SENTRY_DSN') or None
 
     AWS_REGION = os.environ.get('AWS_REGION') or None
+
+    SESSION_COOKIE_SECURE = True
+    SESSION_COOKIE_HTTPONLY = True
+    SESSION_COOKIE_SAMESITE = 'Lax'