pyfedi/app/community/util.py

from datetime import datetime
from typing import List

import requests
from PIL import Image, ImageOps
from flask import request, abort
from flask_login import current_user
from pillow_heif import register_heif_opener

from app import db, cache
from app.constants import POST_TYPE_ARTICLE, POST_TYPE_LINK, POST_TYPE_IMAGE
from app.models import Community, File, BannedInstances, PostReply, PostVote, Post, utcnow
from app.utils import get_request, gibberish, markdown_to_html, domain_from_url, validate_image
from sqlalchemy import desc, text
import os
from opengraph_parse import parse_page


allowed_extensions = ['.gif', '.jpg', '.jpeg', '.png', '.webp', '.heic']

def search_for_community(address: str):
    if address.startswith('!'):
        name, server = address[1:].split('@')

        banned = BannedInstances.query.filter_by(domain=server).first()
        if banned:
            reason = f" Reason: {banned.reason}" if banned.reason is not None else ''
            raise Exception(f"{server} is blocked.{reason}")  # todo: create custom exception class hierarchy

        already_exists = Community.query.filter_by(ap_id=address[1:]).first()
        if already_exists:
            return already_exists

        # Look up the profile address of the community using WebFinger
        # todo: try, except block around every get_request
        webfinger_data = get_request(f"https://{server}/.well-known/webfinger",
                                     params={'resource': f"acct:{address[1:]}"})
        if webfinger_data.status_code == 200:
            webfinger_json = webfinger_data.json()
            for links in webfinger_json['links']:
                if 'rel' in links and links['rel'] == 'self':  # this contains the URL of the activitypub profile
                    type = links['type'] if 'type' in links else 'application/activity+json'
                    # retrieve the activitypub profile
                    community_data = get_request(links['href'], headers={'Accept': type})
                    # to see the structure of the json contained in community_data, do a GET to https://lemmy.world/c/technology with header Accept: application/activity+json
                    if community_data.status_code == 200:
                        community_json = community_data.json()
                        if community_json['type'] == 'Group':
                            community = Community(name=community_json['preferredUsername'],
                                                  title=community_json['name'],
                                                  description=community_json['summary'],
                                                  nsfw=community_json['sensitive'],
                                                  restricted_to_mods=community_json['postingRestrictedToMods'],
                                                  created_at=community_json['published'],
                                                  last_active=community_json['updated'],
                                                  ap_id=f"{address[1:]}",
                                                  ap_public_url=community_json['id'],
                                                  ap_profile_id=community_json['id'],
                                                  ap_followers_url=community_json['followers'],
                                                  ap_inbox_url=community_json['endpoints']['sharedInbox'],
                                                  ap_moderators_url=community_json['attributedTo'] if 'attributedTo' in community_json else None,
                                                  ap_fetched_at=utcnow(),
                                                  ap_domain=server,
                                                  public_key=community_json['publicKey']['publicKeyPem'],
                                                  # language=community_json['language'][0]['identifier'] # todo: language
                                                  )
                            if 'icon' in community_json:
                                # todo: retrieve icon, save to disk, save more complete File record
                                icon = File(source_url=community_json['icon']['url'])
                                community.icon = icon
                                db.session.add(icon)
                            if 'image' in community_json:
                                # todo: retrieve image, save to disk, save more complete File record
                                image = File(source_url=community_json['image']['url'])
                                community.image = image
                                db.session.add(image)
                            db.session.add(community)
                            db.session.commit()
                            return community
        return None


def community_url_exists(url) -> bool:
    community = Community.query.filter_by(ap_profile_id=url).first()
    return community is not None


def actor_to_community(actor) -> Community:
    actor = actor.strip()
    if '@' in actor:
        community = Community.query.filter_by(banned=False, ap_id=actor).first()
    else:
        community = Community.query.filter_by(name=actor, banned=False, ap_id=None).first()
    return community


def ensure_directory_exists(directory):
    parts = directory.split('/')
    rebuild_directory = ''
    for part in parts:
        rebuild_directory += part
        if not os.path.isdir(rebuild_directory):
            os.mkdir(rebuild_directory)
        rebuild_directory += '/'


@cache.memoize(timeout=50)
def opengraph_parse(url):
    try:
        return parse_page(url)
    except Exception as ex:
        return None


def url_to_thumbnail_file(filename) -> File:
    unused, file_extension = os.path.splitext(filename)
    response = requests.get(filename, timeout=5)
    if response.status_code == 200:
        new_filename = gibberish(15)
        directory = 'app/static/media/posts/' + new_filename[0:2] + '/' + new_filename[2:4]
        ensure_directory_exists(directory)
        final_place = os.path.join(directory, new_filename + file_extension)
        with open(final_place, 'wb') as f:
            f.write(response.content)
        with Image.open(final_place) as img:
            img = ImageOps.exif_transpose(img)
            img.thumbnail((150, 150))
            img.save(final_place)
            thumbnail_width = img.width
            thumbnail_height = img.height
        return File(file_name=new_filename + file_extension, thumbnail_width=thumbnail_width,
                    thumbnail_height=thumbnail_height, thumbnail_path=final_place,
                    source_url=filename)


def save_post(form, post: Post):
    post.nsfw = form.nsfw.data
    post.nsfl = form.nsfl.data
    post.notify_author = form.notify_author.data
    if form.type.data == '' or form.type.data == 'discussion':
        post.title = form.discussion_title.data
        post.body = form.discussion_body.data
        post.body_html = markdown_to_html(post.body)
        post.type = POST_TYPE_ARTICLE
    elif form.type.data == 'link':
        post.title = form.link_title.data
        url_changed = post.id is None or form.link_url.data != post.url
        post.url = form.link_url.data
        post.type = POST_TYPE_LINK
        domain = domain_from_url(form.link_url.data)
        domain.post_count += 1
        post.domain = domain

        if url_changed:
            if post.image_id:
                remove_old_file(post.image_id)
                post.image_id = None
            valid_extensions = {'.jpg', '.jpeg', '.png', '.gif', '.webp'}
            unused, file_extension = os.path.splitext(form.link_url.data)  # do not use _ here instead of 'unused'
            # this url is a link to an image - generate a thumbnail of it
            if file_extension in valid_extensions:
                file = url_to_thumbnail_file(form.link_url.data)
                if file:
                    post.image = file
                    db.session.add(file)
            else:
                # check opengraph tags on the page and make a thumbnail if an image is available in the og:image meta tag
                opengraph = opengraph_parse(form.link_url.data)
                if opengraph and opengraph.get('og:image', '') != '':
                    filename = opengraph.get('og:image')
                    valid_extensions = {'.jpg', '.jpeg', '.png', '.gif', '.webp'}
                    unused, file_extension = os.path.splitext(filename)
                    if file_extension.lower() in valid_extensions:
                        file = url_to_thumbnail_file(filename)
                        if file:
                            file.alt_text = opengraph.get('og:title')
                            post.image = file
                            db.session.add(file)

    elif form.type.data == 'image':
        post.title = form.image_title.data
        post.type = POST_TYPE_IMAGE
        uploaded_file = request.files['image_file']
        if uploaded_file and uploaded_file.filename != '':
            if post.image_id:
                remove_old_file(post.image_id)
                post.image_id = None

            # check if this is an allowed type of file
            file_ext = os.path.splitext(uploaded_file.filename)[1]
            if file_ext.lower() not in allowed_extensions or file_ext != validate_image(
                    uploaded_file.stream):
                abort(400)
            new_filename = gibberish(15)

            # set up the storage directory
            directory = 'app/static/media/posts/' + new_filename[0:2] + '/' + new_filename[2:4]
            ensure_directory_exists(directory)

            # save the file
            final_place = os.path.join(directory, new_filename + file_ext)
            final_place_thumbnail = os.path.join(directory, new_filename + '_thumbnail.webp')
            uploaded_file.save(final_place)

            if file_ext.lower() == '.heic':
                register_heif_opener()

            # resize if necessary
            img = Image.open(final_place)
            img = ImageOps.exif_transpose(img)
            img_width = img.width
            img_height = img.height
            if img.width > 2000 or img.height > 2000:
                img.thumbnail((2000, 2000))
                img.save(final_place)
                img_width = img.width
                img_height = img.height
            # save a second, smaller, version as a thumbnail
            img.thumbnail((256, 256))
            img.save(final_place_thumbnail, format="WebP", quality=93)
            thumbnail_width = img.width
            thumbnail_height = img.height

            file = File(file_path=final_place, file_name=new_filename + file_ext, alt_text=form.image_title.data,
                        width=img_width, height=img_height, thumbnail_width=thumbnail_width,
                        thumbnail_height=thumbnail_height, thumbnail_path=final_place_thumbnail)
            post.image = file
            db.session.add(file)

    elif form.type.data == 'poll':
        ...
    else:
        raise Exception('invalid post type')
    if post.id is None:
        postvote = PostVote(user_id=current_user.id, author_id=current_user.id, post=post, effect=1.0)
        post.up_votes = 1
        post.score = 1
        db.session.add(postvote)
        db.session.add(post)


def remove_old_file(file_id):
    remove_file = File.query.get(file_id)
    remove_file.delete_from_disk()


def save_icon_file(icon_file) -> File:
    # check if this is an allowed type of file
    file_ext = os.path.splitext(icon_file.filename)[1]
    if file_ext.lower() not in allowed_extensions or file_ext != validate_image(
            icon_file.stream):
        abort(400)
    new_filename = gibberish(15)

    # set up the storage directory
    directory = 'app/static/media/communities/' + new_filename[0:2] + '/' + new_filename[2:4]
    ensure_directory_exists(directory)

    # save the file
    final_place = os.path.join(directory, new_filename + file_ext)
    final_place_thumbnail = os.path.join(directory, new_filename + '_thumbnail.webp')
    icon_file.save(final_place)

    if file_ext.lower() == '.heic':
        register_heif_opener()

    # resize if necessary
    img = Image.open(final_place)
    img = ImageOps.exif_transpose(img)
    img_width = img.width
    img_height = img.height
    if img.width > 200 or img.height > 200:
        img.thumbnail((200, 200))
        img.save(final_place)
        img_width = img.width
        img_height = img.height
    # save a second, smaller, version as a thumbnail
    img.thumbnail((32, 32))
    img.save(final_place_thumbnail, format="WebP", quality=93)
    thumbnail_width = img.width
    thumbnail_height = img.height

    file = File(file_path=final_place, file_name=new_filename + file_ext, alt_text='community icon',
                width=img_width, height=img_height, thumbnail_width=thumbnail_width,
                thumbnail_height=thumbnail_height, thumbnail_path=final_place_thumbnail)
    db.session.add(file)
    return file


def save_banner_file(banner_file) -> File:
    # check if this is an allowed type of file
    file_ext = os.path.splitext(banner_file.filename)[1]
    if file_ext.lower() not in allowed_extensions or file_ext != validate_image(
            banner_file.stream):
        abort(400)
    new_filename = gibberish(15)

    # set up the storage directory
    directory = 'app/static/media/communities/' + new_filename[0:2] + '/' + new_filename[2:4]
    ensure_directory_exists(directory)

    # save the file
    final_place = os.path.join(directory, new_filename + file_ext)
    final_place_thumbnail = os.path.join(directory, new_filename + '_thumbnail.webp')
    banner_file.save(final_place)

    if file_ext.lower() == '.heic':
        register_heif_opener()

    # resize if necessary
    img = Image.open(final_place)
    img = ImageOps.exif_transpose(img)
    img_width = img.width
    img_height = img.height
    if img.width > 1000 or img.height > 300:
        img.thumbnail((1000, 300))
        img.save(final_place)
        img_width = img.width
        img_height = img.height

    file = File(file_path=final_place, file_name=new_filename + file_ext, alt_text='community banner',
                width=img_width, height=img_height)
    db.session.add(file)
    return file