refactor PostData class

* Split up into SearchForm class as part of searching module
* Split up into WatchState class as part of index module

tubearchivist/home/src/index.py

@@ -15,7 +15,7 @@ import requests
 import yt_dlp as youtube_dl
 from bs4 import BeautifulSoup
 from home.src.config import AppConfig
-from home.src.helper import DurationConverter, clean_string
+from home.src.helper import DurationConverter, clean_string, process_url_list
 
 
 class YoutubeChannel:
@@ -363,6 +363,78 @@ class YoutubeVideo:
             print(response.text)
 
 
+class WatchState:
+    """handle watched checkbox for videos and channels"""
+
+    CONFIG = AppConfig().config
+    ES_URL = CONFIG["application"]["es_url"]
+    HEADERS = {"Content-type": "application/json"}
+
+    def __init__(self, youtube_id):
+        self.youtube_id = youtube_id
+        self.stamp = int(datetime.now().strftime("%s"))
+
+    def mark_as_watched(self):
+        """update es with new watched value"""
+        url_type = self.dedect_type()
+        if url_type == "video":
+            self.mark_vid_watched()
+        elif url_type == "channel":
+            self.mark_channel_watched()
+
+        print(f"marked {self.youtube_id} as watched")
+
+    def dedect_type(self):
+        """find youtube id type"""
+        url_process = process_url_list([self.youtube_id])
+        url_type = url_process[0]["type"]
+
+        return url_type
+
+    def mark_vid_watched(self):
+        """change watched status of single video"""
+        url = self.ES_URL + "/ta_video/_update/" + self.youtube_id
+        data = {
+            "doc": {"player": {"watched": True, "watched_date": self.stamp}}
+        }
+        payload = json.dumps(data)
+        request = requests.post(url, data=payload, headers=self.HEADERS)
+        if not request.ok:
+            print(request.text)
+
+    def mark_channel_watched(self):
+        """change watched status of every video in channel"""
+        es_url = self.ES_URL
+        headers = self.HEADERS
+        youtube_id = self.youtube_id
+        # create pipeline
+        data = {
+            "description": youtube_id,
+            "processors": [
+                {"set": {"field": "player.watched", "value": True}},
+                {"set": {"field": "player.watched_date", "value": self.stamp}},
+            ],
+        }
+        payload = json.dumps(data)
+        url = f"{es_url}/_ingest/pipeline/{youtube_id}"
+        request = requests.put(url, data=payload, headers=headers)
+        if not request.ok:
+            print(request.text)
+            raise ValueError("failed to post ingest pipeline")
+
+        # apply pipeline
+        must_list = [
+            {"term": {"channel.channel_id": {"value": youtube_id}}},
+            {"term": {"player.watched": {"value": False}}},
+        ]
+        data = {"query": {"bool": {"must": must_list}}}
+        payload = json.dumps(data)
+        url = f"{es_url}/ta_video/_update_by_query?pipeline={youtube_id}"
+        request = requests.post(url, data=payload, headers=headers)
+        if not request.ok:
+            print(request.text)
+
+
 def index_new_video(youtube_id, missing_vid=False):
     """combine video and channel classes for new video index"""
     vid_handler = YoutubeVideo(youtube_id)

tubearchivist/home/src/searching.py

@@ -182,6 +182,40 @@ class SearchHandler:
         return hit
 
 
+class SearchForm:
+    """build query from search form data"""
+
+    CONFIG = AppConfig().config
+    ES_URL = CONFIG["application"]["es_url"]
+
+    def search_channels(self, search_query):
+        """fancy searching channels as you type"""
+        url = self.ES_URL + "/ta_channel/_search"
+        data = {
+            "size": 10,
+            "query": {
+                "multi_match": {
+                    "query": search_query,
+                    "type": "bool_prefix",
+                    "fields": [
+                        "channel_name.search_as_you_type",
+                        "channel_name._2gram",
+                        "channel_name._3gram",
+                    ],
+                }
+            },
+        }
+        look_up = SearchHandler(url, data, cache=False)
+        search_results = look_up.get_data()
+        return {"results": search_results}
+
+    @staticmethod
+    def search_videos():
+        """searching for videos"""
+        # TBD palceholder for now
+        return False
+
+
 class Pagination:
     """
     figure out the pagination based on page size and total_hits

tubearchivist/home/views.py

@@ -6,10 +6,8 @@ Functionality:
 
 import json
 import urllib.parse
-from datetime import datetime
 from time import sleep
 
-import requests
 from django.http import JsonResponse
 from django.shortcuts import redirect, render
 from django.utils.http import urlencode
@@ -22,7 +20,8 @@ from home.src.helper import (
     process_url_list,
     set_message,
 )
-from home.src.searching import Pagination, SearchHandler
+from home.src.index import WatchState
+from home.src.searching import Pagination, SearchForm, SearchHandler
 from home.tasks import (
     download_pending,
     download_single,
@@ -450,7 +449,7 @@ def process(request):
     if request.method == "POST":
         post_dict = json.loads(request.body.decode())
         post_handler = PostData(post_dict)
-        if post_handler.to_do:
+        if post_handler.to_exec:
             task_result = post_handler.run_task()
             return JsonResponse(task_result)
 
@@ -458,183 +457,128 @@ def process(request):
 
 
 class PostData:
-    """generic post handler from process route"""
+    """
-
+    map frontend http post values to backend funcs
-    CONFIG = AppConfig().config
+    handover long running tasks to celery
-    ES_URL = CONFIG["application"]["es_url"]
+    """
-
-    VALID_KEYS = [
-        "watched",
-        "rescan_pending",
-        "ignore",
-        "dl_pending",
-        "unsubscribe",
-        "sort_order",
-        "hide_watched",
-        "show_subed_only",
-        "channel-search",
-        "video-search",
-        "dlnow",
-        "manual-import",
-        "db-backup",
-        "db-restore",
-    ]
 
     def __init__(self, post_dict):
         self.post_dict = post_dict
-        self.to_do = self.validate()
+        self.to_exec, self.exec_val = list(post_dict.items())[0]
-
-    def validate(self):
-        """validate the post_dict"""
-        to_do = []
-        for key, value in self.post_dict.items():
-            if key in self.VALID_KEYS:
-                task_item = {"task": key, "status": value}
-                print(task_item)
-                to_do.append(task_item)
-            else:
-                print(key + " not a valid key")
-
-        return to_do
 
     def run_task(self):
-        """run through the tasks to do"""
+        """execute and return task result"""
-        for item in self.to_do:
+        to_exec = self.exec_map()
-            task = item["task"]
+        task_result = to_exec()
-            if task == "watched":
+        return task_result
-                youtube_id = item["status"]
+
-                self.parse_watched(youtube_id)
+    def exec_map(self):
-            elif task == "rescan_pending":
+        """map dict key and return function to execute"""
-                print("rescan subscribed channels")
+        exec_map = {
-                update_subscribed.delay()
+            "watched": self.watched,
-            elif task == "ignore":
+            "rescan_pending": self.rescan_pending,
-                print("ignore video")
+            "ignore": self.ignore,
-                handler = PendingList()
+            "dl_pending": self.dl_pending,
-                ignore_list = item["status"]
+            "unsubscribe": self.unsubscribe,
-                handler.ignore_from_pending([ignore_list])
+            "sort_order": self.sort_order,
-            elif task == "dl_pending":
+            "hide_watched": self.hide_watched,
-                print("download pending")
+            "show_subed_only": self.show_subed_only,
-                download_pending.delay()
+            "dlnow": self.dlnow,
-            elif task == "unsubscribe":
+            "manual-import": self.manual_import,
-                channel_id_unsub = item["status"]
+            "db-backup": self.db_backup,
-                print("unsubscribe from " + channel_id_unsub)
+            "db-restore": self.db_restore,
-                ChannelSubscription().change_subscribe(
+            "channel-search": self.channel_search,
-                    channel_id_unsub, channel_subscribed=False
+        }
-                )
+
-            elif task == "sort_order":
+        return exec_map[self.to_exec]
-                sort_order = item["status"]
+
-                set_message("sort_order", sort_order, expire=False)
+    def watched(self):
-            elif task == "hide_watched":
+        """mark as watched"""
-                hide_watched = bool(int(item["status"]))
+        WatchState(self.exec_val).mark_as_watched()
-                print(item["status"])
-                set_message("hide_watched", hide_watched, expire=False)
-            elif task == "show_subed_only":
-                show_subed_only = bool(int(item["status"]))
-                print(show_subed_only)
-                set_message("show_subed_only", show_subed_only, expire=False)
-            elif task == "channel-search":
-                search_query = item["status"]
-                print("searching for: " + search_query)
-                search_results = self.search_channels(search_query)
-                return search_results
-            elif task == "video-search":
-                search_query = item["status"]
-                print("searching for: " + search_query)
-                search_results = self.search_videos(search_query)
-                return search_results
-            elif task == "dlnow":
-                youtube_id = item["status"]
-                print("downloading: " + youtube_id)
-                download_single.delay(youtube_id=youtube_id)
-            elif task == "manual-import":
-                print("starting manual import")
-                run_manual_import.delay()
-            elif task == "db-backup":
-                print("backing up database")
-                run_backup.delay()
-            elif task == "db-restore":
-                print("restoring index from backup zip")
-                run_restore_backup.delay()
         return {"success": True}
 
-    def search_channels(self, search_query):
+    @staticmethod
-        """fancy searching channels as you type"""
+    def rescan_pending():
-        url = self.ES_URL + "/ta_channel/_search"
+        """look for new items in subscribed channels"""
-        data = {
+        print("rescan subscribed channels")
-            "size": 10,
+        update_subscribed.delay()
-            "query": {
+        return {"success": True}
-                "multi_match": {
-                    "query": search_query,
-                    "type": "bool_prefix",
-                    "fields": [
-                        "channel_name.search_as_you_type",
-                        "channel_name._2gram",
-                        "channel_name._3gram",
-                    ],
-                }
-            },
-        }
-        look_up = SearchHandler(url, data, cache=False)
-        search_results = look_up.get_data()
-        return {"results": search_results}
 
-    def search_videos(self, search_query):
+    def ignore(self):
-        """fancy searching videos as you type"""
+        """ignore from download queue"""
-        url = self.ES_URL + "/ta_video/_search"
+        print("ignore video")
-        data = {
+        handler = PendingList()
-            "size": 10,
+        ignore_list = self.exec_val
-            "query": {
+        handler.ignore_from_pending([ignore_list])
-                "multi_match": {
+        return {"success": True}
-                    "query": search_query,
-                    "type": "bool_prefix",
-                    "fields": [
-                        "title.search_as_you_type",
-                        "title._2gram",
-                        "title._3gram",
-                    ],
-                }
-            },
-        }
-        look_up = SearchHandler(url, data, cache=False)
-        search_results = look_up.get_data()
-        return {"results": search_results}
 
-    def parse_watched(self, youtube_id):
+    @staticmethod
-        """marked as watched based on id type"""
+    def dl_pending():
-        es_url = self.ES_URL
+        """start the download queue"""
-        id_type = process_url_list([youtube_id])[0]["type"]
+        print("download pending")
-        stamp = int(datetime.now().strftime("%s"))
+        download_pending.delay()
-        if id_type == "video":
+        return {"success": True}
-            stamp = int(datetime.now().strftime("%s"))
+
-            url = self.ES_URL + "/ta_video/_update/" + youtube_id
+    def unsubscribe(self):
-            source = {
+        """unsubscribe from channel"""
-                "doc": {"player": {"watched": True, "watched_date": stamp}}
+        channel_id_unsub = self.exec_val
-            }
+        print("unsubscribe from " + channel_id_unsub)
-            request = requests.post(url, json=source)
+        ChannelSubscription().change_subscribe(
-            if not request.ok:
+            channel_id_unsub, channel_subscribed=False
-                print(request.text)
+        )
-        elif id_type == "channel":
+        return {"success": True}
-            headers = {"Content-type": "application/json"}
+
-            data = {
+    def sort_order(self):
-                "description": youtube_id,
+        """change the sort between published to downloaded"""
-                "processors": [
+        sort_order = self.exec_val
-                    {"set": {"field": "player.watched", "value": True}},
+        set_message("sort_order", sort_order, expire=False)
-                    {"set": {"field": "player.watched_date", "value": stamp}},
+        return {"success": True}
-                ],
+
-            }
+    def hide_watched(self):
-            payload = json.dumps(data)
+        """toggle if to show watched vids or not"""
-            url = es_url + "/_ingest/pipeline/" + youtube_id
+        hide_watched = bool(int(self.exec_val))
-            request = requests.put(url, data=payload, headers=headers)
+        print(f"hide watched: {hide_watched}")
-            if not request.ok:
+        set_message("hide_watched", hide_watched, expire=False)
-                print(request.text)
+        return {"success": True}
-            # apply pipeline
+
-            must_list = [
+    def show_subed_only(self):
-                {"term": {"channel.channel_id": {"value": youtube_id}}},
+        """show or hide subscribed channels only on channels page"""
-                {"term": {"player.watched": {"value": False}}},
+        show_subed_only = bool(int(self.exec_val))
-            ]
+        print(f"show subed only: {show_subed_only}")
-            data = {"query": {"bool": {"must": must_list}}}
+        set_message("show_subed_only", show_subed_only, expire=False)
-            payload = json.dumps(data)
+        return {"success": True}
-            url = f"{es_url}/ta_video/_update_by_query?pipeline={youtube_id}"
+
-            request = requests.post(url, data=payload, headers=headers)
+    def dlnow(self):
-            if not request.ok:
+        """start downloading single vid now"""
-                print(request.text)
+        youtube_id = self.exec_val
+        print("downloading: " + youtube_id)
+        download_single.delay(youtube_id=youtube_id)
+        return {"success": True}
+
+    @staticmethod
+    def manual_import():
+        """run manual import from settings page"""
+        print("starting manual import")
+        run_manual_import.delay()
+        return {"success": True}
+
+    @staticmethod
+    def db_backup():
+        """backup es to zip from settings page"""
+        print("backing up database")
+        run_backup.delay()
+        return {"success": True}
+
+    @staticmethod
+    def db_restore():
+        """restore es zip from settings page"""
+        print("restoring index from backup zip")
+        run_restore_backup.delay()
+        return {"success": True}
+
+    def channel_search(self):
+        """search for channel name as_you_type"""
+        search_query = self.exec_val
+        print("searching for: " + search_query)
+        search_results = SearchForm().search_channels(search_query)
+        return search_results