split download app

tubearchivist/api/src/search_processor.py

@@ -6,7 +6,7 @@ Functionality:
 
 import urllib.parse
 
-from home.src.download.thumbnails import ThumbManager
+from download.src.thumbnails import ThumbManager
 from home.src.ta.helper import date_parser, get_duration_str
 from home.src.ta.settings import EnvironmentSettings
 

tubearchivist/api/urls.py

@@ -6,16 +6,6 @@ from django.urls import path
 urlpatterns = [
     path("ping/", views.PingView.as_view(), name="ping"),
     path("login/", views.LoginApiView.as_view(), name="api-login"),
-    path(
-        "download/",
-        views.DownloadApiListView.as_view(),
-        name="api-download-list",
-    ),
-    path(
-        "download/<slug:video_id>/",
-        views.DownloadApiView.as_view(),
-        name="api-download",
-    ),
     path(
         "refresh/",
         views.RefreshView.as_view(),

tubearchivist/api/views.py

@@ -10,9 +10,8 @@ from api.src.aggs import (
     WatchProgress,
 )
 from api.src.search_processor import SearchProcess
+from download.src.yt_dlp_base import CookieHandler
 from home.models import CustomPeriodicTask
-from home.src.download.queue import PendingInteract
-from home.src.download.yt_dlp_base import CookieHandler
 from home.src.es.backup import ElasticBackup
 from home.src.es.connect import ElasticWrap
 from home.src.es.snapshot import ElasticSnapshot
@@ -27,12 +26,7 @@ from home.src.ta.ta_redis import RedisArchivist
 from home.src.ta.task_config import TASK_CONFIG
 from home.src.ta.task_manager import TaskCommand, TaskManager
 from home.src.ta.users import UserConfig
-from home.tasks import (
+from home.tasks import check_reindex, run_restore_backup
-    check_reindex,
-    download_pending,
-    extrac_dl,
-    run_restore_backup,
-)
 from rest_framework import permissions
 from rest_framework.authentication import (
     SessionAuthentication,
@@ -129,127 +123,6 @@ class ApiBaseView(APIView):
             self.response["paginate"] = self.pagination_handler.pagination
 
 
-class DownloadApiView(ApiBaseView):
-    """resolves to /api/download/<video_id>/
-    GET: returns metadata dict of an item in the download queue
-    POST: update status of item to pending or ignore
-    DELETE: forget from download queue
-    """
-
-    search_base = "ta_download/_doc/"
-    valid_status = ["pending", "ignore", "ignore-force", "priority"]
-    permission_classes = [AdminOnly]
-
-    def get(self, request, video_id):
-        # pylint: disable=unused-argument
-        """get request"""
-        self.get_document(video_id)
-        return Response(self.response, status=self.status_code)
-
-    def post(self, request, video_id):
-        """post to video to change status"""
-        item_status = request.data.get("status")
-        if item_status not in self.valid_status:
-            message = f"{video_id}: invalid status {item_status}"
-            print(message)
-            return Response({"message": message}, status=400)
-
-        if item_status == "ignore-force":
-            extrac_dl.delay(video_id, status="ignore")
-            message = f"{video_id}: set status to ignore"
-            return Response(request.data)
-
-        _, status_code = PendingInteract(video_id).get_item()
-        if status_code == 404:
-            message = f"{video_id}: item not found {status_code}"
-            return Response({"message": message}, status=404)
-
-        print(f"{video_id}: change status to {item_status}")
-        PendingInteract(video_id, item_status).update_status()
-        if item_status == "priority":
-            download_pending.delay(auto_only=True)
-
-        return Response(request.data)
-
-    @staticmethod
-    def delete(request, video_id):
-        # pylint: disable=unused-argument
-        """delete single video from queue"""
-        print(f"{video_id}: delete from queue")
-        PendingInteract(video_id).delete_item()
-
-        return Response({"success": True})
-
-
-class DownloadApiListView(ApiBaseView):
-    """resolves to /api/download/
-    GET: returns latest videos in the download queue
-    POST: add a list of videos to download queue
-    DELETE: remove items based on query filter
-    """
-
-    search_base = "ta_download/_search/"
-    valid_filter = ["pending", "ignore"]
-    permission_classes = [AdminOnly]
-
-    def get(self, request):
-        """get request"""
-        query_filter = request.GET.get("filter", False)
-        self.data.update({"sort": [{"timestamp": {"order": "asc"}}]})
-
-        must_list = []
-        if query_filter:
-            if query_filter not in self.valid_filter:
-                message = f"invalid url query filter: {query_filter}"
-                print(message)
-                return Response({"message": message}, status=400)
-
-            must_list.append({"term": {"status": {"value": query_filter}}})
-
-        filter_channel = request.GET.get("channel", False)
-        if filter_channel:
-            must_list.append(
-                {"term": {"channel_id": {"value": filter_channel}}}
-            )
-
-        self.data["query"] = {"bool": {"must": must_list}}
-
-        self.get_document_list(request)
-        return Response(self.response)
-
-    @staticmethod
-    def post(request):
-        """add list of videos to download queue"""
-        data = request.data
-        auto_start = bool(request.GET.get("autostart"))
-        try:
-            to_add = data["data"]
-        except KeyError:
-            message = "missing expected data key"
-            print(message)
-            return Response({"message": message}, status=400)
-
-        pending = [i["youtube_id"] for i in to_add if i["status"] == "pending"]
-        url_str = " ".join(pending)
-        extrac_dl.delay(url_str, auto_start=auto_start)
-
-        return Response(data)
-
-    def delete(self, request):
-        """delete download queue"""
-        query_filter = request.GET.get("filter", False)
-        if query_filter not in self.valid_filter:
-            message = f"invalid url query filter: {query_filter}"
-            print(message)
-            return Response({"message": message}, status=400)
-
-        message = f"delete queue by status: {query_filter}"
-        print(message)
-        PendingInteract(status=query_filter).delete_by_status()
-
-        return Response({"message": message})
-
-
 class PingView(ApiBaseView):
     """resolves to /api/ping/
     GET: test your connection

tubearchivist/channel/src/index.py

@@ -8,8 +8,8 @@ import json
 import os
 from datetime import datetime
 
-from home.src.download.thumbnails import ThumbManager
+from download.src.thumbnails import ThumbManager
-from home.src.download.yt_dlp_base import YtWrap
+from download.src.yt_dlp_base import YtWrap
 from home.src.es.connect import ElasticWrap, IndexPaginate
 from home.src.index.generic import YouTubeItem
 from home.src.ta.settings import EnvironmentSettings

tubearchivist/channel/views.py

@@ -2,7 +2,7 @@
 
 from api.views import AdminWriteOnly, ApiBaseView
 from channel.src.index import YoutubeChannel
-from home.src.download.subscriptions import ChannelSubscription
+from download.src.subscriptions import ChannelSubscription
 from home.src.ta.urlparser import Parser
 from home.tasks import subscribe_to
 from rest_framework.response import Response

tubearchivist/config/settings.py

@@ -65,6 +65,7 @@ INSTALLED_APPS = [
     "video",
     "channel",
     "playlist",
+    "download",
     "config",
 ]
 

tubearchivist/config/urls.py

@@ -23,5 +23,6 @@ urlpatterns = [
     path("api/video/", include("video.urls")),
     path("api/channel/", include("channel.urls")),
     path("api/playlist/", include("playlist.urls")),
+    path("api/download/", include("download.urls")),
     path("admin/", admin.site.urls),
 ]

tubearchivist/download/src/queue.py

@@ -7,9 +7,9 @@ Functionality:
 import json
 from datetime import datetime
 
-from home.src.download.subscriptions import ChannelSubscription
+from download.src.subscriptions import ChannelSubscription
-from home.src.download.thumbnails import ThumbManager
+from download.src.thumbnails import ThumbManager
-from home.src.download.yt_dlp_base import YtWrap
+from download.src.yt_dlp_base import YtWrap
 from home.src.es.connect import ElasticWrap, IndexPaginate
 from home.src.ta.config import AppConfig
 from home.src.ta.helper import get_duration_str, is_shorts

tubearchivist/download/src/subscriptions.py

@@ -5,8 +5,8 @@ Functionality:
 """
 
 from channel.src.index import YoutubeChannel
-from home.src.download.thumbnails import ThumbManager
+from download.src.thumbnails import ThumbManager
-from home.src.download.yt_dlp_base import YtWrap
+from download.src.yt_dlp_base import YtWrap
 from home.src.es.connect import IndexPaginate
 from home.src.ta.config import AppConfig
 from home.src.ta.helper import is_missing

tubearchivist/download/src/yt_dlp_handler.py

@@ -11,9 +11,9 @@ import shutil
 from datetime import datetime
 
 from channel.src.index import YoutubeChannel
-from home.src.download.queue import PendingList
+from download.src.queue import PendingList
-from home.src.download.subscriptions import PlaylistSubscription
+from download.src.subscriptions import PlaylistSubscription
-from home.src.download.yt_dlp_base import YtWrap
+from download.src.yt_dlp_base import YtWrap
 from home.src.es.connect import ElasticWrap, IndexPaginate
 from home.src.ta.config import AppConfig
 from home.src.ta.helper import get_channel_overwrites, ignore_filelist

tubearchivist/download/urls.py

@@ -0,0 +1,13 @@
+"""all download API urls"""
+
+from django.urls import path
+from download import views
+
+urlpatterns = [
+    path("", views.DownloadApiListView.as_view(), name="api-download-list"),
+    path(
+        "<slug:video_id>/",
+        views.DownloadApiView.as_view(),
+        name="api-download",
+    ),
+]

tubearchivist/download/views.py

@@ -0,0 +1,127 @@
+"""all download API views"""
+
+from api.views import AdminOnly, ApiBaseView
+from download.src.queue import PendingInteract
+from home.tasks import download_pending, extrac_dl
+from rest_framework.response import Response
+
+
+class DownloadApiListView(ApiBaseView):
+    """resolves to /api/download/
+    GET: returns latest videos in the download queue
+    POST: add a list of videos to download queue
+    DELETE: remove items based on query filter
+    """
+
+    search_base = "ta_download/_search/"
+    valid_filter = ["pending", "ignore"]
+    permission_classes = [AdminOnly]
+
+    def get(self, request):
+        """get request"""
+        query_filter = request.GET.get("filter", False)
+        self.data.update({"sort": [{"timestamp": {"order": "asc"}}]})
+
+        must_list = []
+        if query_filter:
+            if query_filter not in self.valid_filter:
+                message = f"invalid url query filter: {query_filter}"
+                print(message)
+                return Response({"message": message}, status=400)
+
+            must_list.append({"term": {"status": {"value": query_filter}}})
+
+        filter_channel = request.GET.get("channel", False)
+        if filter_channel:
+            must_list.append(
+                {"term": {"channel_id": {"value": filter_channel}}}
+            )
+
+        self.data["query"] = {"bool": {"must": must_list}}
+
+        self.get_document_list(request)
+        return Response(self.response)
+
+    @staticmethod
+    def post(request):
+        """add list of videos to download queue"""
+        data = request.data
+        auto_start = bool(request.GET.get("autostart"))
+        try:
+            to_add = data["data"]
+        except KeyError:
+            message = "missing expected data key"
+            print(message)
+            return Response({"message": message}, status=400)
+
+        pending = [i["youtube_id"] for i in to_add if i["status"] == "pending"]
+        url_str = " ".join(pending)
+        extrac_dl.delay(url_str, auto_start=auto_start)
+
+        return Response(data)
+
+    def delete(self, request):
+        """delete download queue"""
+        query_filter = request.GET.get("filter", False)
+        if query_filter not in self.valid_filter:
+            message = f"invalid url query filter: {query_filter}"
+            print(message)
+            return Response({"message": message}, status=400)
+
+        message = f"delete queue by status: {query_filter}"
+        print(message)
+        PendingInteract(status=query_filter).delete_by_status()
+
+        return Response({"message": message})
+
+
+class DownloadApiView(ApiBaseView):
+    """resolves to /api/download/<video_id>/
+    GET: returns metadata dict of an item in the download queue
+    POST: update status of item to pending or ignore
+    DELETE: forget from download queue
+    """
+
+    search_base = "ta_download/_doc/"
+    valid_status = ["pending", "ignore", "ignore-force", "priority"]
+    permission_classes = [AdminOnly]
+
+    def get(self, request, video_id):
+        # pylint: disable=unused-argument
+        """get request"""
+        self.get_document(video_id)
+        return Response(self.response, status=self.status_code)
+
+    def post(self, request, video_id):
+        """post to video to change status"""
+        item_status = request.data.get("status")
+        if item_status not in self.valid_status:
+            message = f"{video_id}: invalid status {item_status}"
+            print(message)
+            return Response({"message": message}, status=400)
+
+        if item_status == "ignore-force":
+            extrac_dl.delay(video_id, status="ignore")
+            message = f"{video_id}: set status to ignore"
+            return Response(request.data)
+
+        _, status_code = PendingInteract(video_id).get_item()
+        if status_code == 404:
+            message = f"{video_id}: item not found {status_code}"
+            return Response({"message": message}, status=404)
+
+        print(f"{video_id}: change status to {item_status}")
+        PendingInteract(video_id, item_status).update_status()
+        if item_status == "priority":
+            download_pending.delay(auto_only=True)
+
+        return Response(request.data)
+
+    @staticmethod
+    def delete(request, video_id):
+        # pylint: disable=unused-argument
+        """delete single video from queue"""
+        print(f"{video_id}: delete from queue")
+        PendingInteract(video_id).delete_item()
+
+        return Response({"success": True})

tubearchivist/home/src/index/generic.py

@@ -5,7 +5,7 @@ functionality:
 
 import math
 
-from home.src.download.yt_dlp_base import YtWrap
+from download.src.yt_dlp_base import YtWrap
 from home.src.es.connect import ElasticWrap
 from home.src.ta.config import AppConfig
 from home.src.ta.users import UserConfig
@@ -104,8 +104,8 @@ class Pagination:
     def first_guess(self):
         """build first guess before api call"""
         page_get = self.page_get
+        page_from = 0
         if page_get in [0, 1]:
-            page_from = 0
             prev_pages = False
         elif page_get > 1:
             page_from = (page_get - 1) * self.page_size

tubearchivist/home/src/index/manual.py

@@ -11,7 +11,7 @@ import re
 import shutil
 import subprocess
 
-from home.src.download.thumbnails import ThumbManager
+from download.src.thumbnails import ThumbManager
 from home.src.ta.config import AppConfig
 from home.src.ta.helper import ignore_filelist
 from home.src.ta.settings import EnvironmentSettings

tubearchivist/home/src/index/reindex.py

@@ -11,10 +11,10 @@ from time import sleep
 from typing import Callable, TypedDict
 
 from channel.src.index import YoutubeChannel
+from download.src.subscriptions import ChannelSubscription
+from download.src.thumbnails import ThumbManager
+from download.src.yt_dlp_base import CookieHandler
 from home.models import CustomPeriodicTask
-from home.src.download.subscriptions import ChannelSubscription
-from home.src.download.thumbnails import ThumbManager
-from home.src.download.yt_dlp_base import CookieHandler
 from home.src.es.connect import ElasticWrap, IndexPaginate
 from home.src.ta.config import AppConfig
 from home.src.ta.settings import EnvironmentSettings

tubearchivist/home/src/ta/urlparser.py

@@ -6,7 +6,7 @@ Functionality:
 
 from urllib.parse import parse_qs, urlparse
 
-from home.src.download.yt_dlp_base import YtWrap
+from download.src.yt_dlp_base import YtWrap
 from video.src.constants import VideoTypeEnum
 
 

tubearchivist/home/tasks.py

@@ -9,13 +9,10 @@ Functionality:
 from celery import Task, shared_task
 from celery.exceptions import Retry
 from channel.src.index import YoutubeChannel
-from home.src.download.queue import PendingList
+from download.src.queue import PendingList
-from home.src.download.subscriptions import (
+from download.src.subscriptions import SubscriptionHandler, SubscriptionScanner
-    SubscriptionHandler,
+from download.src.thumbnails import ThumbFilesystem, ThumbValidator
-    SubscriptionScanner,
+from download.src.yt_dlp_handler import VideoDownloader
-)
-from home.src.download.thumbnails import ThumbFilesystem, ThumbValidator
-from home.src.download.yt_dlp_handler import VideoDownloader
 from home.src.es.backup import ElasticBackup
 from home.src.es.index_setup import ElasitIndexWrap
 from home.src.index.filesystem import Scanner

tubearchivist/home/views.py

@@ -20,9 +20,9 @@ from django.http import Http404
 from django.shortcuts import redirect, render
 from django.utils.decorators import method_decorator
 from django.views import View
+from download.src.queue import PendingInteract
+from download.src.yt_dlp_base import CookieHandler
 from home.models import CustomPeriodicTask
-from home.src.download.queue import PendingInteract
-from home.src.download.yt_dlp_base import CookieHandler
 from home.src.es.backup import ElasticBackup
 from home.src.es.connect import ElasticWrap
 from home.src.es.snapshot import ElasticSnapshot

tubearchivist/playlist/src/index.py

@@ -8,7 +8,7 @@ import json
 from datetime import datetime
 
 from channel.src import index as channel
-from home.src.download.thumbnails import ThumbManager
+from download.src.thumbnails import ThumbManager
 from home.src.es.connect import ElasticWrap, IndexPaginate
 from home.src.index.generic import YouTubeItem
 from video.src.index import YoutubeVideo

tubearchivist/playlist/views.py

@@ -1,7 +1,7 @@
 """all playlist API views"""
 
 from api.views import AdminWriteOnly, ApiBaseView
-from home.src.download.subscriptions import PlaylistSubscription
+from download.src.subscriptions import PlaylistSubscription
 from home.src.ta.users import UserConfig
 from home.tasks import subscribe_to
 from playlist.src.index import YoutubePlaylist

tubearchivist/video/src/comments.py

@@ -7,7 +7,7 @@ Functionality:
 
 from datetime import datetime
 
-from home.src.download.yt_dlp_base import YtWrap
+from download.src.yt_dlp_base import YtWrap
 from home.src.es.connect import ElasticWrap
 from home.src.ta.config import AppConfig
 from home.src.ta.ta_redis import RedisQueue