From 52be7bad5d863eecd14ace3a13c1ba3b45c505b0 Mon Sep 17 00:00:00 2001 From: rimu <3310831+rimu@users.noreply.github.com> Date: Sat, 14 Dec 2024 14:24:21 +1300 Subject: [PATCH 1/4] switch to requests.get in generate_image_from_video_url #373 --- app/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/utils.py b/app/utils.py index 6b3bb600..14bfdc06 100644 --- a/app/utils.py +++ b/app/utils.py @@ -17,6 +17,7 @@ import math from urllib.parse import urlparse, parse_qs, urlencode from functools import wraps import flask +import requests from bs4 import BeautifulSoup, MarkupResemblesLocatorWarning import warnings import jwt @@ -1111,7 +1112,7 @@ def in_sorted_list(arr, target): # Makes a still image from a video url, without downloading the whole video file def generate_image_from_video_url(video_url, output_path, length=2): - response = httpx_client.get(video_url, stream=True, timeout=5, + response = requests.get(video_url, stream=True, timeout=5, headers={'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:127.0) Gecko/20100101 Firefox/127.0'}) # Imgur requires a user agent content_type = response.headers.get('Content-Type') if content_type: From a5881261907c635247a8a20c0f884a6ca0ecc13d Mon Sep 17 00:00:00 2001 From: freamon Date: Sat, 14 Dec 2024 11:52:18 +0000 Subject: [PATCH 2/4] Exclude extra cross-posts from main feed #206 --- app/main/routes.py | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/app/main/routes.py b/app/main/routes.py index aebf502c..85ab6f59 100644 --- a/app/main/routes.py +++ b/app/main/routes.py @@ -116,9 +116,39 @@ def home_page(sort, view_filter): posts = posts.order_by(desc(Post.last_active)) # Pagination - posts = posts.paginate(page=page, per_page=100 if current_user.is_authenticated and not low_bandwidth else 50, error_out=False) - next_url = url_for('main.index', page=posts.next_num, sort=sort, view_filter=view_filter) if posts.has_next else None - prev_url = url_for('main.index', page=posts.prev_num, sort=sort, view_filter=view_filter) if posts.has_prev and page != 1 else None + if view_filter == 'subscribed' and current_user.is_authenticated and sort == 'new': + # use python list instead of DB query + posts = posts.all() + + # exclude extra cross-posts from feed + already_seen = [] + limit = 100 if not low_bandwidth else 50 + #i = -1 # option 1: don't exclude cross-posts + #i = limit - 1 # option 2: exclude cross-posts from the first page only + i = (limit * 10) - 1 # option 3: exclude cross-posts across a 'magic number' of pages + #i = len(posts) - 1 # option 4: exclude all cross-posts ever + while i >= 0: + if not posts[i].cross_posts: + i -= 1 + continue + if posts[i].id in already_seen: + posts.pop(i) + else: + already_seen.extend(posts[i].cross_posts) + i -= 1 + + # paginate manually (can't use paginate()) + start = (page - 1) * limit + end = start + limit + posts = posts[start:end] + next_page = page + 1 if len(posts) == limit else None + previous_page = page - 1 if page != 1 else None + next_url = url_for('main.index', page=next_page, sort=sort, view_filter=view_filter) if next_page else None + prev_url = url_for('main.index', page=previous_page, sort=sort, view_filter=view_filter) if previous_page else None + else: + posts = posts.paginate(page=page, per_page=100 if current_user.is_authenticated and not low_bandwidth else 50, error_out=False) + next_url = url_for('main.index', page=posts.next_num, sort=sort, view_filter=view_filter) if posts.has_next else None + prev_url = url_for('main.index', page=posts.prev_num, sort=sort, view_filter=view_filter) if posts.has_prev and page != 1 else None # Active Communities active_communities = Community.query.filter_by(banned=False) From 46c150cecf385c64fb58946c16fd8dc3a7084170 Mon Sep 17 00:00:00 2001 From: freamon Date: Sat, 14 Dec 2024 13:05:09 +0000 Subject: [PATCH 3/4] Undo changes from previous commit until debugged --- app/main/routes.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/app/main/routes.py b/app/main/routes.py index 85ab6f59..df7b04ab 100644 --- a/app/main/routes.py +++ b/app/main/routes.py @@ -116,6 +116,7 @@ def home_page(sort, view_filter): posts = posts.order_by(desc(Post.last_active)) # Pagination + """ if view_filter == 'subscribed' and current_user.is_authenticated and sort == 'new': # use python list instead of DB query posts = posts.all() @@ -146,9 +147,10 @@ def home_page(sort, view_filter): next_url = url_for('main.index', page=next_page, sort=sort, view_filter=view_filter) if next_page else None prev_url = url_for('main.index', page=previous_page, sort=sort, view_filter=view_filter) if previous_page else None else: - posts = posts.paginate(page=page, per_page=100 if current_user.is_authenticated and not low_bandwidth else 50, error_out=False) - next_url = url_for('main.index', page=posts.next_num, sort=sort, view_filter=view_filter) if posts.has_next else None - prev_url = url_for('main.index', page=posts.prev_num, sort=sort, view_filter=view_filter) if posts.has_prev and page != 1 else None + """ + posts = posts.paginate(page=page, per_page=100 if current_user.is_authenticated and not low_bandwidth else 50, error_out=False) + next_url = url_for('main.index', page=posts.next_num, sort=sort, view_filter=view_filter) if posts.has_next else None + prev_url = url_for('main.index', page=posts.prev_num, sort=sort, view_filter=view_filter) if posts.has_prev and page != 1 else None # Active Communities active_communities = Community.query.filter_by(banned=False) From d4e9ff14d96b3814de8fe6a994b85b089f9d0825 Mon Sep 17 00:00:00 2001 From: freamon Date: Sat, 14 Dec 2024 14:55:14 +0000 Subject: [PATCH 4/4] bugfix for extra cross-posts exclusion commit --- app/main/routes.py | 12 +++++------- app/templates/index.html | 6 +++--- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/app/main/routes.py b/app/main/routes.py index df7b04ab..ba64cf6e 100644 --- a/app/main/routes.py +++ b/app/main/routes.py @@ -116,7 +116,6 @@ def home_page(sort, view_filter): posts = posts.order_by(desc(Post.last_active)) # Pagination - """ if view_filter == 'subscribed' and current_user.is_authenticated and sort == 'new': # use python list instead of DB query posts = posts.all() @@ -125,8 +124,8 @@ def home_page(sort, view_filter): already_seen = [] limit = 100 if not low_bandwidth else 50 #i = -1 # option 1: don't exclude cross-posts - #i = limit - 1 # option 2: exclude cross-posts from the first page only - i = (limit * 10) - 1 # option 3: exclude cross-posts across a 'magic number' of pages + #i = min(limit - 1, len(posts) - 1) # option 2: exclude cross-posts from the first page only + i = min((limit * 10) - 1, len(posts) - 1) # option 3: exclude cross-posts across a 'magic number' of pages #i = len(posts) - 1 # option 4: exclude all cross-posts ever while i >= 0: if not posts[i].cross_posts: @@ -147,10 +146,9 @@ def home_page(sort, view_filter): next_url = url_for('main.index', page=next_page, sort=sort, view_filter=view_filter) if next_page else None prev_url = url_for('main.index', page=previous_page, sort=sort, view_filter=view_filter) if previous_page else None else: - """ - posts = posts.paginate(page=page, per_page=100 if current_user.is_authenticated and not low_bandwidth else 50, error_out=False) - next_url = url_for('main.index', page=posts.next_num, sort=sort, view_filter=view_filter) if posts.has_next else None - prev_url = url_for('main.index', page=posts.prev_num, sort=sort, view_filter=view_filter) if posts.has_prev and page != 1 else None + posts = posts.paginate(page=page, per_page=100 if current_user.is_authenticated and not low_bandwidth else 50, error_out=False) + next_url = url_for('main.index', page=posts.next_num, sort=sort, view_filter=view_filter) if posts.has_next else None + prev_url = url_for('main.index', page=posts.prev_num, sort=sort, view_filter=view_filter) if posts.has_prev and page != 1 else None # Active Communities active_communities = Community.query.filter_by(banned=False) diff --git a/app/templates/index.html b/app/templates/index.html index 9ea7c2ae..8398d712 100644 --- a/app/templates/index.html +++ b/app/templates/index.html @@ -15,12 +15,12 @@ {% include "_home_nav.html" %} {% include "_view_filter_nav.html" %}
- {% for post in posts.items -%} + {% for post in posts %} {% include 'post/_post_teaser.html' %} - {% else -%} + {% else %}

{{ _('No posts yet. Join some communities to see more.') }}

{{ _('More communities') }}

- {% endfor -%} + {% endfor %}