From 49db71e05d3e856d87d4be055ffb02ee8733adfd Mon Sep 17 00:00:00 2001
From: rimu <3310831+rimu@users.noreply.github.com>
Date: Tue, 12 Mar 2024 20:58:47 +1300
Subject: [PATCH] opt out of search

---
 app/activitypub/util.py      | 15 +++++++++++++--
 app/community/util.py        |  1 +
 app/models.py                |  2 +-
 app/post/routes.py           |  1 +
 app/search/routes.py         | 25 +++++++++++++++++--------
 app/templates/base.html      |  3 +++
 app/templates/post/post.html |  4 ++--
 app/user/forms.py            |  2 +-
 app/user/routes.py           |  6 ++++++
 9 files changed, 45 insertions(+), 14 deletions(-)

diff --git a/app/activitypub/util.py b/app/activitypub/util.py
index 683c35f7..425f5125 100644
--- a/app/activitypub/util.py
+++ b/app/activitypub/util.py
@@ -339,10 +339,19 @@ def refresh_user_profile_task(user_id):
         if actor_data.status_code == 200:
             activity_json = actor_data.json()
             actor_data.close()
+
+            # update indexible state on their posts, if necessary
+            new_indexable = activity_json['indexable'] if 'indexable' in activity_json else True
+            if new_indexable != user.indexable:
+                db.session.execute(text('UPDATE "post" set indexable = :indexable WHERE user_id = :user_id'),
+                                   {'user_id': user.id,
+                                    'indexable': new_indexable})
+
             user.user_name = activity_json['preferredUsername']
             user.about_html = parse_summary(activity_json)
             user.ap_fetched_at = utcnow()
-            user.public_key=activity_json['publicKey']['publicKeyPem']
+            user.public_key = activity_json['publicKey']['publicKeyPem']
+            user.indexable = new_indexable
 
             avatar_changed = cover_changed = False
             if 'icon' in activity_json:
@@ -594,6 +603,7 @@ def post_json_to_model(post_json, user, community) -> Post:
                     last_active=post_json['published'],
                     instance_id=user.instance_id
                     )
+        post.indexable = user.indexable
         if 'source' in post_json and \
                 post_json['source']['mediaType'] == 'text/markdown':
             post.body = post_json['source']['content']
@@ -1212,7 +1222,8 @@ def create_post(activity_log: ActivityPubLog, community: Community, request_json
                 type=constants.POST_TYPE_ARTICLE,
                 up_votes=1,
                 score=instance_weight(user.ap_domain),
-                instance_id=user.instance_id
+                instance_id=user.instance_id,
+                indexable=user.indexable
                 )
     # Get post content. Lemmy and Kbin put this in different places.
     if 'source' in request_json['object'] and isinstance(request_json['object']['source'], dict) and request_json['object']['source']['mediaType'] == 'text/markdown': # Lemmy
diff --git a/app/community/util.py b/app/community/util.py
index 75132ded..f86e5811 100644
--- a/app/community/util.py
+++ b/app/community/util.py
@@ -164,6 +164,7 @@ def url_to_thumbnail_file(filename) -> File:
 
 
 def save_post(form, post: Post):
+    post.indexable = current_user.indexable
     post.nsfw = form.nsfw.data
     post.nsfl = form.nsfl.data
     post.notify_author = form.notify_author.data
diff --git a/app/models.py b/app/models.py
index c42f8a59..69449bd5 100644
--- a/app/models.py
+++ b/app/models.py
@@ -776,7 +776,7 @@ class Post(db.Model):
     nsfl = db.Column(db.Boolean, default=False, index=True)
     sticky = db.Column(db.Boolean, default=False)
     notify_author = db.Column(db.Boolean, default=True)
-    indexable = db.Column(db.Boolean, default=False)
+    indexable = db.Column(db.Boolean, default=True)
     from_bot = db.Column(db.Boolean, default=False, index=True)
     created_at = db.Column(db.DateTime, index=True, default=utcnow)    # this is when the content arrived here
     posted_at = db.Column(db.DateTime, index=True, default=utcnow)     # this is when the original server created it
diff --git a/app/post/routes.py b/app/post/routes.py
index 0c5ed613..67472b84 100644
--- a/app/post/routes.py
+++ b/app/post/routes.py
@@ -216,6 +216,7 @@ def show_post(post_id: int):
                            canonical=post.ap_id, form=form, replies=replies, THREAD_CUTOFF_DEPTH=constants.THREAD_CUTOFF_DEPTH,
                            description=description, og_image=og_image, POST_TYPE_IMAGE=constants.POST_TYPE_IMAGE,
                            POST_TYPE_LINK=constants.POST_TYPE_LINK, POST_TYPE_ARTICLE=constants.POST_TYPE_ARTICLE,
+                           noindex=not post.author.indexable,
                            etag=f"{post.id}{sort}_{hash(post.last_active)}", markdown_editor=current_user.is_authenticated and current_user.markdown_editor,
                            low_bandwidth=request.cookies.get('low_bandwidth', '0') == '1', SUBSCRIPTION_MEMBER=SUBSCRIPTION_MEMBER,
                            moderating_communities=moderating_communities(current_user.get_id()),
diff --git a/app/search/routes.py b/app/search/routes.py
index e4a96cf8..8edbcef7 100644
--- a/app/search/routes.py
+++ b/app/search/routes.py
@@ -5,11 +5,10 @@ from sqlalchemy import or_
 
 from app.models import Post
 from app.search import bp
-from app.utils import moderating_communities, joined_communities, render_template, blocked_domains
+from app.utils import moderating_communities, joined_communities, render_template, blocked_domains, blocked_instances
 
 
 @bp.route('/search', methods=['GET', 'POST'])
-@login_required
 def run_search():
     if request.args.get('q') is not None:
         q = request.args.get('q')
@@ -17,16 +16,26 @@ def run_search():
         low_bandwidth = request.cookies.get('low_bandwidth', '0') == '1'
 
         posts = Post.query.search(q)
-        if current_user.ignore_bots:
+        if current_user.is_authenticated:
+            if current_user.ignore_bots:
+                posts = posts.filter(Post.from_bot == False)
+            if current_user.show_nsfl is False:
+                posts = posts.filter(Post.nsfl == False)
+            if current_user.show_nsfw is False:
+                posts = posts.filter(Post.nsfw == False)
+            domains_ids = blocked_domains(current_user.id)
+            if domains_ids:
+                posts = posts.filter(or_(Post.domain_id.not_in(domains_ids), Post.domain_id == None))
+            instance_ids = blocked_instances(current_user.id)
+            if instance_ids:
+                posts = posts.filter(or_(Post.instance_id.not_in(instance_ids), Post.instance_id == None))
+        else:
             posts = posts.filter(Post.from_bot == False)
-        if current_user.show_nsfl is False:
             posts = posts.filter(Post.nsfl == False)
-        if current_user.show_nsfw is False:
             posts = posts.filter(Post.nsfw == False)
 
-        domains_ids = blocked_domains(current_user.id)
-        if domains_ids:
-            posts = posts.filter(or_(Post.domain_id.not_in(domains_ids), Post.domain_id == None))
+        posts = posts.filter(Post.indexable == True)
+
         posts = posts.paginate(page=page, per_page=100 if current_user.is_authenticated and not low_bandwidth else 50,
                                error_out=False)
 
diff --git a/app/templates/base.html b/app/templates/base.html
index 92c8e266..7e76319b 100644
--- a/app/templates/base.html
+++ b/app/templates/base.html
@@ -70,6 +70,9 @@
         {% if rss_feed %}
             <link rel="alternate" type="application/rss+xml" title="{{ rss_feed_name }}" href="{{ rss_feed }}" />
         {% endif %}
+        {% if noindex %}
+            <meta name="robots" content="noindex">
+        {% endif %}
         <script nonce="{{ session['nonce']}}">
             const getStoredTheme = () => localStorage.getItem('theme')
             const setStoredTheme = theme => localStorage.setItem('theme', theme)
diff --git a/app/templates/post/post.html b/app/templates/post/post.html
index 6e13545f..1ba56c72 100644
--- a/app/templates/post/post.html
+++ b/app/templates/post/post.html
@@ -75,7 +75,7 @@
                 {% macro render_comment(comment) %}
                     <div id="comment_{{ comment['comment'].id }}" class="comment {% if comment['comment'].score <= -10 %}low_score{% endif %}
                         {% if comment['comment'].author.id == post.author.id %}original_poster{% endif %}" aria-level="{{ comment['comment'].depth + 1 }}" role="treeitem" aria-expanded="true" tabindex="0">
-                        <div class="limit_height">
+                        <div class="limit_height">{% if not comment['comment'].author.indexable %}<!--googleoff: all-->{% endif %}
                             <div class="comment_author">
                                 {% if comment['comment'].author.deleted %}
                                     [deleted]
@@ -107,7 +107,7 @@
                             </div>
                             <div class="comment_body hidable {% if comment['comment'].reports and current_user.is_authenticated and post.community.is_moderator(current_user) %}reported{% endif %}">
                                 {{ comment['comment'].body_html | safe }}
-                            </div>
+                            </div>{% if not comment['comment'].author.indexable %}<!--googleon: all-->{% endif %}
                         </div>
                         <div class="comment_actions hidable">
                             {% if post.comments_enabled %}
diff --git a/app/user/forms.py b/app/user/forms.py
index 7b23b13c..d80760fb 100644
--- a/app/user/forms.py
+++ b/app/user/forms.py
@@ -38,7 +38,7 @@ class SettingsForm(FlaskForm):
     nsfl = BooleanField(_l('Show NSFL posts'))
     markdown_editor = BooleanField(_l('Use markdown editor GUI when writing'))
     searchable = BooleanField(_l('Show profile in user list'))
-    indexable = BooleanField(_l('Allow search engines to index this profile'))
+    indexable = BooleanField(_l('My posts appear in search results'))
     manually_approves_followers = BooleanField(_l('Manually approve followers'))
     import_file = FileField(_('Import community subscriptions and user blocks from Lemmy'))
     sorts = [('hot', _l('Hot')),
diff --git a/app/user/routes.py b/app/user/routes.py
index 78d0e774..c1a35b44 100644
--- a/app/user/routes.py
+++ b/app/user/routes.py
@@ -79,6 +79,7 @@ def show_profile(user):
                            description=description, subscribed=subscribed, upvoted=upvoted,
                            post_next_url=post_next_url, post_prev_url=post_prev_url,
                            replies_next_url=replies_next_url, replies_prev_url=replies_prev_url,
+                           noindex=not user.indexable,
                            moderating_communities=moderating_communities(current_user.get_id()),
                            joined_communities=joined_communities(current_user.get_id())
                            )
@@ -159,6 +160,7 @@ def change_settings():
     form = SettingsForm()
     form.theme.choices = theme_list()
     if form.validate_on_submit():
+        propagate_indexable = form.indexable.data != current_user.indexable
         current_user.newsletter = form.newsletter.data
         current_user.ignore_bots = form.ignore_bots.data
         current_user.show_nsfw = form.nsfw.data
@@ -170,6 +172,10 @@ def change_settings():
         current_user.email_unread = form.email_unread.data
         current_user.markdown_editor = form.markdown_editor.data
         import_file = request.files['import_file']
+        if propagate_indexable:
+            db.session.execute(text('UPDATE "post" set indexable = :indexable WHERE user_id = :user_id'),
+                               {'user_id': current_user.id,
+                                'indexable': current_user.indexable})
         if import_file and import_file.filename != '':
             file_ext = os.path.splitext(import_file.filename)[1]
             if file_ext.lower() != '.json':