tubearchivist/tubearchivist/home/views.py

"""
Functionality:
- all views for home app
- process post data received from frontend via ajax
"""

import json
import urllib.parse
from datetime import datetime
from time import sleep

import requests
from django.http import JsonResponse
from django.shortcuts import redirect, render
from django.utils.http import urlencode
from django.views import View
from home.src.config import AppConfig
from home.src.download import ChannelSubscription, PendingList
from home.src.helper import (get_dl_message, get_message, process_url_list,
                             set_message)
from home.src.searching import Pagination, SearchHandler
from home.tasks import (download_pending, download_single, extrac_dl,
                        run_backup, run_manual_import, run_restore_backup,
                        update_subscribed)


class HomeView(View):
    """ resolves to /
    handle home page and video search post functionality
    """

    CONFIG = AppConfig().config
    ES_URL = CONFIG['application']['es_url']

    def get(self, request):
        """ return home search results """
        colors, sort_order, hide_watched = self.read_config()
        # handle search
        search_get = request.GET.get('search', False)
        if search_get:
            search_encoded = urllib.parse.quote(search_get)
        else:
            search_encoded = False
        # define page size
        page_get = int(request.GET.get('page', 0))
        pagination_handler = Pagination(page_get, search_encoded)

        url = self.ES_URL + '/ta_video/_search'

        data = self.build_data(
            pagination_handler, sort_order, search_get, hide_watched
        )

        search = SearchHandler(url, data)
        videos_hits = search.get_data()
        max_hits = search.max_hits
        pagination_handler.validate(max_hits)
        context = {
            'videos': videos_hits,
            'pagination': pagination_handler.pagination,
            'sortorder': sort_order,
            'hide_watched': hide_watched,
            'colors': colors
        }
        return render(request, 'home/home.html', context)

    @staticmethod
    def build_data(pagination_handler, sort_order, search_get, hide_watched):
        """ build the data dict for the search query """
        page_size = pagination_handler.pagination['page_size']
        page_from = pagination_handler.pagination['page_from']
        data = {
            "size": page_size, "from": page_from, "query": {"match_all": {}},
            "sort": [
                {"published": {"order": "desc"}},
                {"date_downloaded": {"order": "desc"}}
            ]
        }
        # define sort
        if sort_order == 'downloaded':
            del data['sort'][0]
        if search_get:
            del data['sort']
        if hide_watched:
            data['query'] = {"term": {"player.watched": {"value": False}}}
        if search_get:
            query = {
                "multi_match": {
                    "query": search_get,
                    "fields": ["title", "channel.channel_name", "tags"],
                    "type": "cross_fields",
                    "operator": "and"
                }
            }
            data['query'] = query

        return data

    @staticmethod
    def read_config():
        """ read needed values from redis """
        config_handler = AppConfig().config
        colors = config_handler['application']['colors']
        sort_order = get_message('sort_order')
        hide_watched = get_message('hide_watched')
        return colors, sort_order, hide_watched

    @staticmethod
    def post(request):
        """ handle post from search form """
        post_data = dict(request.POST)
        search_query = post_data['videoSearch'][0]
        search_url = '/?' + urlencode({'search': search_query})
        return redirect(search_url, permanent=True)


class AboutView(View):
    """ resolves to /about/
    show helpful how to information
    """

    @staticmethod
    def get(request):
        """ handle http get """
        config = AppConfig().config
        colors = config['application']['colors']
        context = {
            'title': 'About',
            'colors': colors
        }
        return render(request, 'home/about.html', context)


class DownloadView(View):
    """ resolves to /download/
    takes POST for downloading youtube links
    """

    def get(self, request):
        """ handle get requests """
        config = AppConfig().config
        colors = config['application']['colors']

        page_get = int(request.GET.get('page', 0))
        pagination_handler = Pagination(page_get)

        url = config['application']['es_url'] + '/ta_download/_search'
        data = self.build_data(pagination_handler)
        search = SearchHandler(url, data, cache=False)

        videos_hits = search.get_data()
        max_hits = search.max_hits

        if videos_hits:
            all_pending = [i['source'] for i in videos_hits]
            pagination_handler.validate(max_hits)
            pagination = pagination_handler.pagination
        else:
            all_pending = False
            pagination = False

        context = {
            'pending': all_pending,
            'max_hits': max_hits,
            'pagination': pagination,
            'title': 'Downloads',
            'colors': colors
        }
        return render(request, 'home/downloads.html', context)

    @staticmethod
    def build_data(pagination_handler):
        """ build data dict for search """
        page_size = pagination_handler.pagination['page_size']
        page_from = pagination_handler.pagination['page_from']
        data = {
            "size": page_size, "from": page_from,
            "query": {"term": {"status": {"value": "pending"}}},
            "sort": [{"timestamp": {"order": "desc"}}]
        }
        return data

    @staticmethod
    def post(request):
        """ handle post requests """
        download_post = dict(request.POST)
        if 'vid-url' in download_post.keys():
            url_str = download_post['vid-url']
            print('adding to queue')
            youtube_ids = process_url_list(url_str)
            if not youtube_ids:
                # failed to process
                print(url_str)
                mess_dict = {
                    "status": "downloading",
                    "level": "error",
                    "title": 'Failed to extract links.',
                    "message": ''
                }
                set_message('progress:download', mess_dict)
                return redirect('downloads')

            print(youtube_ids)
            extrac_dl.delay(youtube_ids)

        sleep(2)
        return redirect('downloads', permanent=True)


class ChannelIdView(View):
    """ resolves to /channel/<channel-id>/
    display single channel page from channel_id
    """

    def get(self, request, channel_id_detail):
        """ get method """
        es_url, colors = self.read_config()
        context = self.get_channel_videos(request, channel_id_detail, es_url)
        context.update({'colors': colors})
        return render(request, 'home/channel_id.html', context)

    @staticmethod
    def read_config():
        """ read config file """
        config = AppConfig().config
        es_url = config['application']['es_url']
        colors = config['application']['colors']
        return es_url, colors

    def get_channel_videos(self, request, channel_id_detail, es_url):
        """ get channel from video index """
        page_get = int(request.GET.get('page', 0))
        pagination_handler = Pagination(page_get)
        # get data
        url = es_url + '/ta_video/_search'
        data = self.build_data(pagination_handler, channel_id_detail)
        search = SearchHandler(url, data)
        videos_hits = search.get_data()
        max_hits = search.max_hits
        if max_hits:
            channel_info = videos_hits[0]['source']['channel']
            channel_name = channel_info['channel_name']
            pagination_handler.validate(max_hits)
            pagination = pagination_handler.pagination
        else:
            # get details from channel index when when no hits
            channel_info, channel_name = self.get_channel_info(
                channel_id_detail, es_url
            )
            videos_hits = False
            pagination = False

        context = {
            'channel_info': channel_info,
            'videos': videos_hits,
            'max_hits': max_hits,
            'pagination': pagination,
            'title': 'Channel: ' + channel_name,
        }

        return context

    @staticmethod
    def build_data(pagination_handler, channel_id_detail):
        """ build data dict for search """
        page_size = pagination_handler.pagination['page_size']
        page_from = pagination_handler.pagination['page_from']
        data = {
            "size": page_size, "from": page_from,
            "query": {
                "term": {"channel.channel_id": {"value": channel_id_detail}}
            },
            "sort": [
                {"published": {"order": "desc"}},
                {"date_downloaded": {"order": "desc"}}
            ]
        }
        return data

    @staticmethod
    def get_channel_info(channel_id_detail, es_url):
        """ get channel info from channel index if no videos """
        url = f'{es_url}/ta_channel/_doc/{channel_id_detail}'
        data = False
        search = SearchHandler(url, data)
        channel_data = search.get_data()
        channel_info = channel_data[0]['source']
        channel_name = channel_info['channel_name']
        return channel_info, channel_name


class ChannelView(View):
    """ resolves to /channel/
    handle functionality for channel overview page, subscribe to channel,
    search as you type for channel name
    """

    def get(self, request):
        """ handle http get requests """
        es_url, colors = self.read_config()
        page_get = int(request.GET.get('page', 0))
        pagination_handler = Pagination(page_get)
        page_size = pagination_handler.pagination['page_size']
        page_from = pagination_handler.pagination['page_from']
        # get
        url = es_url + '/ta_channel/_search'
        data = {
            "size": page_size, "from": page_from, "query": {"match_all": {}},
            "sort": [{"channel_name.keyword": {"order": "asc"}}]
        }
        show_subed_only = get_message('show_subed_only')
        if show_subed_only:
            data['query'] = {"term": {"channel_subscribed": {"value": True}}}
        search = SearchHandler(url, data)
        channel_hits = search.get_data()
        max_hits = search.max_hits
        pagination_handler.validate(search.max_hits)
        context = {
            'channels': channel_hits,
            'max_hits': max_hits,
            'pagination': pagination_handler.pagination,
            'show_subed_only': show_subed_only,
            'title': 'Channels',
            'colors': colors
        }
        return render(request, 'home/channel.html', context)

    @staticmethod
    def read_config():
        """ read config file """
        config = AppConfig().config
        es_url = config['application']['es_url']
        colors = config['application']['colors']
        return es_url, colors

    def post(self, request):
        """ handle http post requests """
        subscriptions_post = dict(request.POST)
        print(subscriptions_post)
        subscriptions_post = dict(request.POST)
        if 'subscribe' in subscriptions_post.keys():
            sub_str = subscriptions_post['subscribe']
            try:
                youtube_ids = process_url_list(sub_str)
                self.subscribe_to(youtube_ids)
            except ValueError:
                print('parsing subscribe ids failed!')
                print(sub_str)

        sleep(1)
        return redirect('channel', permanent=True)

    @staticmethod
    def subscribe_to(youtube_ids):
        """ process the subscribe ids """
        for youtube_id in youtube_ids:
            if youtube_id['type'] == 'video':
                to_sub = youtube_id['url']
                vid_details = PendingList().get_youtube_details(to_sub)
                channel_id_sub = vid_details['channel_id']
            elif youtube_id['type'] == 'channel':
                channel_id_sub = youtube_id['url']
            else:
                raise ValueError('failed to subscribe to: ' + youtube_id)

            ChannelSubscription().change_subscribe(
                channel_id_sub, channel_subscribed=True
            )
            print('subscribed to: ' + channel_id_sub)


class VideoView(View):
    """ resolves to /video/<video-id>/
    display details about a single video
    """

    def get(self, request, video_id):
        """ get single video """
        es_url, colors = self.read_config()
        url = f'{es_url}/ta_video/_doc/{video_id}'
        data = None
        look_up = SearchHandler(url, data)
        video_hit = look_up.get_data()
        video_data = video_hit[0]['source']
        video_title = video_data['title']
        context = {
            'video': video_data,
            'title': video_title,
            'colors': colors
        }
        return render(request, 'home/video.html', context)

    @staticmethod
    def read_config():
        """ read config file """
        config = AppConfig().config
        es_url = config['application']['es_url']
        colors = config['application']['colors']
        return es_url, colors


class SettingsView(View):
    """ resolves to /settings/
    handle the settings page, display current settings,
    take post request from the form to update settings
    """

    @staticmethod
    def get(request):
        """ read and display current settings """
        config = AppConfig().config
        colors = config['application']['colors']

        context = {
            'title': 'Settings',
            'config': config,
            'colors': colors
        }

        return render(request, 'home/settings.html', context)

    @staticmethod
    def post(request):
        """ handle form post to update settings """
        form_post = dict(request.POST)
        del form_post['csrfmiddlewaretoken']
        print(form_post)
        config_handler = AppConfig()
        config_handler.update_config(form_post)

        return redirect('settings', permanent=True)


def progress(request):
    # pylint: disable=unused-argument
    """ endpoint for download progress ajax calls """
    config = AppConfig().config
    cache_dir = config['application']['cache_dir']
    json_data = get_dl_message(cache_dir)
    return JsonResponse(json_data)


def process(request):
    """ handle all the buttons calls via POST ajax """
    if request.method == 'POST':
        post_dict = json.loads(request.body.decode())
        post_handler = PostData(post_dict)
        if post_handler.to_do:
            task_result = post_handler.run_task()
            return JsonResponse(task_result)

    return JsonResponse({'success': False})


class PostData:
    """ generic post handler from process route """

    CONFIG = AppConfig().config
    ES_URL = CONFIG['application']['es_url']

    VALID_KEYS = [
        "watched", "rescan_pending", "ignore", "dl_pending",
        "unsubscribe", "sort_order", "hide_watched", "show_subed_only",
        "channel-search", "video-search", "dlnow", "manual-import",
        "db-backup", "db-restore"
    ]

    def __init__(self, post_dict):
        self.post_dict = post_dict
        self.to_do = self.validate()

    def validate(self):
        """ validate the post_dict """
        to_do = []
        for key, value in self.post_dict.items():
            if key in self.VALID_KEYS:
                task_item = {'task': key, 'status': value}
                print(task_item)
                to_do.append(task_item)
            else:
                print(key + ' not a valid key')

        return to_do

    def run_task(self):
        """ run through the tasks to do """
        for item in self.to_do:
            task = item['task']
            if task == 'watched':
                youtube_id = item['status']
                self.parse_watched(youtube_id)
            elif task == 'rescan_pending':
                print('rescan subscribed channels')
                update_subscribed.delay()
            elif task == 'ignore':
                print('ignore video')
                handler = PendingList()
                ignore_list = item['status']
                handler.ignore_from_pending([ignore_list])
            elif task == 'dl_pending':
                print('download pending')
                download_pending.delay()
            elif task == 'unsubscribe':
                channel_id_unsub = item['status']
                print('unsubscribe from ' + channel_id_unsub)
                ChannelSubscription().change_subscribe(
                    channel_id_unsub, channel_subscribed=False
                )
            elif task == 'sort_order':
                sort_order = item['status']
                set_message('sort_order', sort_order, expire=False)
            elif task == 'hide_watched':
                hide_watched = bool(int(item['status']))
                print(item['status'])
                set_message('hide_watched', hide_watched, expire=False)
            elif task == 'show_subed_only':
                show_subed_only = bool(int(item['status']))
                print(show_subed_only)
                set_message('show_subed_only', show_subed_only, expire=False)
            elif task == 'channel-search':
                search_query = item['status']
                print('searching for: ' + search_query)
                search_results = self.search_channels(search_query)
                return search_results
            elif task == 'video-search':
                search_query = item['status']
                print('searching for: ' + search_query)
                search_results = self.search_videos(search_query)
                return search_results
            elif task == 'dlnow':
                youtube_id = item['status']
                print('downloading: ' + youtube_id)
                download_single.delay(youtube_id=youtube_id)
            elif task == 'manual-import':
                print('starting manual import')
                run_manual_import.delay()
            elif task == 'db-backup':
                print('backing up database')
                run_backup.delay()
            elif task == 'db-restore':
                print('restoring index from backup zip')
                run_restore_backup.delay()
        return {'success': True}

    def search_channels(self, search_query):
        """ fancy searching channels as you type """
        url = self.ES_URL + '/ta_channel/_search'
        data = {
            "size": 10,
            "query": {
                "multi_match": {
                    "query": search_query,
                    "type": "bool_prefix",
                    "fields": [
                        "channel_name.search_as_you_type",
                        "channel_name._2gram",
                        "channel_name._3gram"
                    ]
                }
            }
        }
        look_up = SearchHandler(url, data, cache=False)
        search_results = look_up.get_data()
        return {'results': search_results}

    def search_videos(self, search_query):
        """ fancy searching videos as you type """
        url = self.ES_URL + '/ta_video/_search'
        data = {
            "size": 10,
            "query": {
                "multi_match": {
                    "query": search_query,
                    "type": "bool_prefix",
                    "fields": [
                        "title.search_as_you_type",
                        "title._2gram",
                        "title._3gram"
                    ]
                }
            }
        }
        look_up = SearchHandler(url, data, cache=False)
        search_results = look_up.get_data()
        return {'results': search_results}

    def parse_watched(self, youtube_id):
        """ marked as watched based on id type """
        es_url = self.ES_URL
        id_type = process_url_list([youtube_id])[0]['type']
        stamp = int(datetime.now().strftime("%s"))
        if id_type == 'video':
            stamp = int(datetime.now().strftime("%s"))
            url = self.ES_URL + '/ta_video/_update/' + youtube_id
            source = {
                "doc": {"player": {"watched": True, "watched_date": stamp}}
            }
            request = requests.post(url, json=source)
            if not request.ok:
                print(request.text)
        elif id_type == 'channel':
            headers = {'Content-type': 'application/json'}
            data = {
                "description": youtube_id,
                "processors": [
                    {"set": {"field": "player.watched", "value": True}},
                    {"set": {"field": "player.watched_date", "value": stamp}}
                ]
            }
            payload = json.dumps(data)
            url = es_url + '/_ingest/pipeline/' + youtube_id
            request = requests.put(url, data=payload, headers=headers)
            if not request.ok:
                print(request.text)
            # apply pipeline
            must_list = [
                {"term": {"channel.channel_id": {"value": youtube_id}}},
                {"term": {"player.watched": {"value": False}}}
            ]
            data = {"query": {"bool": {"must": must_list}}}
            payload = json.dumps(data)
            url = f'{es_url}/ta_video/_update_by_query?pipeline={youtube_id}'
            request = requests.post(url, data=payload, headers=headers)
            if not request.ok:
                print(request.text)