diff --git a/tubearchivist/api/src/search_processor.py b/tubearchivist/api/src/search_processor.py index 05993369..13015d78 100644 --- a/tubearchivist/api/src/search_processor.py +++ b/tubearchivist/api/src/search_processor.py @@ -6,7 +6,7 @@ Functionality: import urllib.parse -from home.src.download.thumbnails import ThumbManager +from download.src.thumbnails import ThumbManager from home.src.ta.helper import date_parser, get_duration_str from home.src.ta.settings import EnvironmentSettings diff --git a/tubearchivist/api/urls.py b/tubearchivist/api/urls.py index 82c0b59d..8ef7105d 100644 --- a/tubearchivist/api/urls.py +++ b/tubearchivist/api/urls.py @@ -6,16 +6,6 @@ from django.urls import path urlpatterns = [ path("ping/", views.PingView.as_view(), name="ping"), path("login/", views.LoginApiView.as_view(), name="api-login"), - path( - "download/", - views.DownloadApiListView.as_view(), - name="api-download-list", - ), - path( - "download//", - views.DownloadApiView.as_view(), - name="api-download", - ), path( "refresh/", views.RefreshView.as_view(), diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index 3f9c068c..855e036c 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -10,9 +10,8 @@ from api.src.aggs import ( WatchProgress, ) from api.src.search_processor import SearchProcess +from download.src.yt_dlp_base import CookieHandler from home.models import CustomPeriodicTask -from home.src.download.queue import PendingInteract -from home.src.download.yt_dlp_base import CookieHandler from home.src.es.backup import ElasticBackup from home.src.es.connect import ElasticWrap from home.src.es.snapshot import ElasticSnapshot @@ -27,12 +26,7 @@ from home.src.ta.ta_redis import RedisArchivist from home.src.ta.task_config import TASK_CONFIG from home.src.ta.task_manager import TaskCommand, TaskManager from home.src.ta.users import UserConfig -from home.tasks import ( - check_reindex, - download_pending, - extrac_dl, - run_restore_backup, -) +from home.tasks import check_reindex, run_restore_backup from rest_framework import permissions from rest_framework.authentication import ( SessionAuthentication, @@ -129,127 +123,6 @@ class ApiBaseView(APIView): self.response["paginate"] = self.pagination_handler.pagination -class DownloadApiView(ApiBaseView): - """resolves to /api/download// - GET: returns metadata dict of an item in the download queue - POST: update status of item to pending or ignore - DELETE: forget from download queue - """ - - search_base = "ta_download/_doc/" - valid_status = ["pending", "ignore", "ignore-force", "priority"] - permission_classes = [AdminOnly] - - def get(self, request, video_id): - # pylint: disable=unused-argument - """get request""" - self.get_document(video_id) - return Response(self.response, status=self.status_code) - - def post(self, request, video_id): - """post to video to change status""" - item_status = request.data.get("status") - if item_status not in self.valid_status: - message = f"{video_id}: invalid status {item_status}" - print(message) - return Response({"message": message}, status=400) - - if item_status == "ignore-force": - extrac_dl.delay(video_id, status="ignore") - message = f"{video_id}: set status to ignore" - return Response(request.data) - - _, status_code = PendingInteract(video_id).get_item() - if status_code == 404: - message = f"{video_id}: item not found {status_code}" - return Response({"message": message}, status=404) - - print(f"{video_id}: change status to {item_status}") - PendingInteract(video_id, item_status).update_status() - if item_status == "priority": - download_pending.delay(auto_only=True) - - return Response(request.data) - - @staticmethod - def delete(request, video_id): - # pylint: disable=unused-argument - """delete single video from queue""" - print(f"{video_id}: delete from queue") - PendingInteract(video_id).delete_item() - - return Response({"success": True}) - - -class DownloadApiListView(ApiBaseView): - """resolves to /api/download/ - GET: returns latest videos in the download queue - POST: add a list of videos to download queue - DELETE: remove items based on query filter - """ - - search_base = "ta_download/_search/" - valid_filter = ["pending", "ignore"] - permission_classes = [AdminOnly] - - def get(self, request): - """get request""" - query_filter = request.GET.get("filter", False) - self.data.update({"sort": [{"timestamp": {"order": "asc"}}]}) - - must_list = [] - if query_filter: - if query_filter not in self.valid_filter: - message = f"invalid url query filter: {query_filter}" - print(message) - return Response({"message": message}, status=400) - - must_list.append({"term": {"status": {"value": query_filter}}}) - - filter_channel = request.GET.get("channel", False) - if filter_channel: - must_list.append( - {"term": {"channel_id": {"value": filter_channel}}} - ) - - self.data["query"] = {"bool": {"must": must_list}} - - self.get_document_list(request) - return Response(self.response) - - @staticmethod - def post(request): - """add list of videos to download queue""" - data = request.data - auto_start = bool(request.GET.get("autostart")) - try: - to_add = data["data"] - except KeyError: - message = "missing expected data key" - print(message) - return Response({"message": message}, status=400) - - pending = [i["youtube_id"] for i in to_add if i["status"] == "pending"] - url_str = " ".join(pending) - extrac_dl.delay(url_str, auto_start=auto_start) - - return Response(data) - - def delete(self, request): - """delete download queue""" - query_filter = request.GET.get("filter", False) - if query_filter not in self.valid_filter: - message = f"invalid url query filter: {query_filter}" - print(message) - return Response({"message": message}, status=400) - - message = f"delete queue by status: {query_filter}" - print(message) - PendingInteract(status=query_filter).delete_by_status() - - return Response({"message": message}) - - class PingView(ApiBaseView): """resolves to /api/ping/ GET: test your connection diff --git a/tubearchivist/channel/src/index.py b/tubearchivist/channel/src/index.py index 956ec7f4..f12e4790 100644 --- a/tubearchivist/channel/src/index.py +++ b/tubearchivist/channel/src/index.py @@ -8,8 +8,8 @@ import json import os from datetime import datetime -from home.src.download.thumbnails import ThumbManager -from home.src.download.yt_dlp_base import YtWrap +from download.src.thumbnails import ThumbManager +from download.src.yt_dlp_base import YtWrap from home.src.es.connect import ElasticWrap, IndexPaginate from home.src.index.generic import YouTubeItem from home.src.ta.settings import EnvironmentSettings diff --git a/tubearchivist/channel/views.py b/tubearchivist/channel/views.py index 9f787c1c..d847f5f8 100644 --- a/tubearchivist/channel/views.py +++ b/tubearchivist/channel/views.py @@ -2,7 +2,7 @@ from api.views import AdminWriteOnly, ApiBaseView from channel.src.index import YoutubeChannel -from home.src.download.subscriptions import ChannelSubscription +from download.src.subscriptions import ChannelSubscription from home.src.ta.urlparser import Parser from home.tasks import subscribe_to from rest_framework.response import Response diff --git a/tubearchivist/config/settings.py b/tubearchivist/config/settings.py index 5321eacb..9dcab7aa 100644 --- a/tubearchivist/config/settings.py +++ b/tubearchivist/config/settings.py @@ -65,6 +65,7 @@ INSTALLED_APPS = [ "video", "channel", "playlist", + "download", "config", ] diff --git a/tubearchivist/config/urls.py b/tubearchivist/config/urls.py index 40563133..be08c843 100644 --- a/tubearchivist/config/urls.py +++ b/tubearchivist/config/urls.py @@ -23,5 +23,6 @@ urlpatterns = [ path("api/video/", include("video.urls")), path("api/channel/", include("channel.urls")), path("api/playlist/", include("playlist.urls")), + path("api/download/", include("download.urls")), path("admin/", admin.site.urls), ] diff --git a/tubearchivist/download/__init__.py b/tubearchivist/download/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tubearchivist/download/migrations/__init__.py b/tubearchivist/download/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tubearchivist/download/src/__init__.py b/tubearchivist/download/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tubearchivist/home/src/download/queue.py b/tubearchivist/download/src/queue.py similarity index 98% rename from tubearchivist/home/src/download/queue.py rename to tubearchivist/download/src/queue.py index b91c4489..449276a4 100644 --- a/tubearchivist/home/src/download/queue.py +++ b/tubearchivist/download/src/queue.py @@ -7,9 +7,9 @@ Functionality: import json from datetime import datetime -from home.src.download.subscriptions import ChannelSubscription -from home.src.download.thumbnails import ThumbManager -from home.src.download.yt_dlp_base import YtWrap +from download.src.subscriptions import ChannelSubscription +from download.src.thumbnails import ThumbManager +from download.src.yt_dlp_base import YtWrap from home.src.es.connect import ElasticWrap, IndexPaginate from home.src.ta.config import AppConfig from home.src.ta.helper import get_duration_str, is_shorts diff --git a/tubearchivist/home/src/download/subscriptions.py b/tubearchivist/download/src/subscriptions.py similarity index 99% rename from tubearchivist/home/src/download/subscriptions.py rename to tubearchivist/download/src/subscriptions.py index 72cf029a..a7ce224d 100644 --- a/tubearchivist/home/src/download/subscriptions.py +++ b/tubearchivist/download/src/subscriptions.py @@ -5,8 +5,8 @@ Functionality: """ from channel.src.index import YoutubeChannel -from home.src.download.thumbnails import ThumbManager -from home.src.download.yt_dlp_base import YtWrap +from download.src.thumbnails import ThumbManager +from download.src.yt_dlp_base import YtWrap from home.src.es.connect import IndexPaginate from home.src.ta.config import AppConfig from home.src.ta.helper import is_missing diff --git a/tubearchivist/home/src/download/thumbnails.py b/tubearchivist/download/src/thumbnails.py similarity index 100% rename from tubearchivist/home/src/download/thumbnails.py rename to tubearchivist/download/src/thumbnails.py diff --git a/tubearchivist/home/src/download/yt_dlp_base.py b/tubearchivist/download/src/yt_dlp_base.py similarity index 100% rename from tubearchivist/home/src/download/yt_dlp_base.py rename to tubearchivist/download/src/yt_dlp_base.py diff --git a/tubearchivist/home/src/download/yt_dlp_handler.py b/tubearchivist/download/src/yt_dlp_handler.py similarity index 99% rename from tubearchivist/home/src/download/yt_dlp_handler.py rename to tubearchivist/download/src/yt_dlp_handler.py index edfd17d0..10db58ee 100644 --- a/tubearchivist/home/src/download/yt_dlp_handler.py +++ b/tubearchivist/download/src/yt_dlp_handler.py @@ -11,9 +11,9 @@ import shutil from datetime import datetime from channel.src.index import YoutubeChannel -from home.src.download.queue import PendingList -from home.src.download.subscriptions import PlaylistSubscription -from home.src.download.yt_dlp_base import YtWrap +from download.src.queue import PendingList +from download.src.subscriptions import PlaylistSubscription +from download.src.yt_dlp_base import YtWrap from home.src.es.connect import ElasticWrap, IndexPaginate from home.src.ta.config import AppConfig from home.src.ta.helper import get_channel_overwrites, ignore_filelist diff --git a/tubearchivist/download/urls.py b/tubearchivist/download/urls.py new file mode 100644 index 00000000..2af17f17 --- /dev/null +++ b/tubearchivist/download/urls.py @@ -0,0 +1,13 @@ +"""all download API urls""" + +from django.urls import path +from download import views + +urlpatterns = [ + path("", views.DownloadApiListView.as_view(), name="api-download-list"), + path( + "/", + views.DownloadApiView.as_view(), + name="api-download", + ), +] diff --git a/tubearchivist/download/views.py b/tubearchivist/download/views.py new file mode 100644 index 00000000..ee91be90 --- /dev/null +++ b/tubearchivist/download/views.py @@ -0,0 +1,127 @@ +"""all download API views""" + +from api.views import AdminOnly, ApiBaseView +from download.src.queue import PendingInteract +from home.tasks import download_pending, extrac_dl +from rest_framework.response import Response + + +class DownloadApiListView(ApiBaseView): + """resolves to /api/download/ + GET: returns latest videos in the download queue + POST: add a list of videos to download queue + DELETE: remove items based on query filter + """ + + search_base = "ta_download/_search/" + valid_filter = ["pending", "ignore"] + permission_classes = [AdminOnly] + + def get(self, request): + """get request""" + query_filter = request.GET.get("filter", False) + self.data.update({"sort": [{"timestamp": {"order": "asc"}}]}) + + must_list = [] + if query_filter: + if query_filter not in self.valid_filter: + message = f"invalid url query filter: {query_filter}" + print(message) + return Response({"message": message}, status=400) + + must_list.append({"term": {"status": {"value": query_filter}}}) + + filter_channel = request.GET.get("channel", False) + if filter_channel: + must_list.append( + {"term": {"channel_id": {"value": filter_channel}}} + ) + + self.data["query"] = {"bool": {"must": must_list}} + + self.get_document_list(request) + return Response(self.response) + + @staticmethod + def post(request): + """add list of videos to download queue""" + data = request.data + auto_start = bool(request.GET.get("autostart")) + try: + to_add = data["data"] + except KeyError: + message = "missing expected data key" + print(message) + return Response({"message": message}, status=400) + + pending = [i["youtube_id"] for i in to_add if i["status"] == "pending"] + url_str = " ".join(pending) + extrac_dl.delay(url_str, auto_start=auto_start) + + return Response(data) + + def delete(self, request): + """delete download queue""" + query_filter = request.GET.get("filter", False) + if query_filter not in self.valid_filter: + message = f"invalid url query filter: {query_filter}" + print(message) + return Response({"message": message}, status=400) + + message = f"delete queue by status: {query_filter}" + print(message) + PendingInteract(status=query_filter).delete_by_status() + + return Response({"message": message}) + + +class DownloadApiView(ApiBaseView): + """resolves to /api/download// + GET: returns metadata dict of an item in the download queue + POST: update status of item to pending or ignore + DELETE: forget from download queue + """ + + search_base = "ta_download/_doc/" + valid_status = ["pending", "ignore", "ignore-force", "priority"] + permission_classes = [AdminOnly] + + def get(self, request, video_id): + # pylint: disable=unused-argument + """get request""" + self.get_document(video_id) + return Response(self.response, status=self.status_code) + + def post(self, request, video_id): + """post to video to change status""" + item_status = request.data.get("status") + if item_status not in self.valid_status: + message = f"{video_id}: invalid status {item_status}" + print(message) + return Response({"message": message}, status=400) + + if item_status == "ignore-force": + extrac_dl.delay(video_id, status="ignore") + message = f"{video_id}: set status to ignore" + return Response(request.data) + + _, status_code = PendingInteract(video_id).get_item() + if status_code == 404: + message = f"{video_id}: item not found {status_code}" + return Response({"message": message}, status=404) + + print(f"{video_id}: change status to {item_status}") + PendingInteract(video_id, item_status).update_status() + if item_status == "priority": + download_pending.delay(auto_only=True) + + return Response(request.data) + + @staticmethod + def delete(request, video_id): + # pylint: disable=unused-argument + """delete single video from queue""" + print(f"{video_id}: delete from queue") + PendingInteract(video_id).delete_item() + + return Response({"success": True}) diff --git a/tubearchivist/home/src/index/generic.py b/tubearchivist/home/src/index/generic.py index 8a502bb5..4211de36 100644 --- a/tubearchivist/home/src/index/generic.py +++ b/tubearchivist/home/src/index/generic.py @@ -5,7 +5,7 @@ functionality: import math -from home.src.download.yt_dlp_base import YtWrap +from download.src.yt_dlp_base import YtWrap from home.src.es.connect import ElasticWrap from home.src.ta.config import AppConfig from home.src.ta.users import UserConfig @@ -104,8 +104,8 @@ class Pagination: def first_guess(self): """build first guess before api call""" page_get = self.page_get + page_from = 0 if page_get in [0, 1]: - page_from = 0 prev_pages = False elif page_get > 1: page_from = (page_get - 1) * self.page_size diff --git a/tubearchivist/home/src/index/manual.py b/tubearchivist/home/src/index/manual.py index 8a4a316e..6271bdf9 100644 --- a/tubearchivist/home/src/index/manual.py +++ b/tubearchivist/home/src/index/manual.py @@ -11,7 +11,7 @@ import re import shutil import subprocess -from home.src.download.thumbnails import ThumbManager +from download.src.thumbnails import ThumbManager from home.src.ta.config import AppConfig from home.src.ta.helper import ignore_filelist from home.src.ta.settings import EnvironmentSettings diff --git a/tubearchivist/home/src/index/reindex.py b/tubearchivist/home/src/index/reindex.py index 8947459c..480a3998 100644 --- a/tubearchivist/home/src/index/reindex.py +++ b/tubearchivist/home/src/index/reindex.py @@ -11,10 +11,10 @@ from time import sleep from typing import Callable, TypedDict from channel.src.index import YoutubeChannel +from download.src.subscriptions import ChannelSubscription +from download.src.thumbnails import ThumbManager +from download.src.yt_dlp_base import CookieHandler from home.models import CustomPeriodicTask -from home.src.download.subscriptions import ChannelSubscription -from home.src.download.thumbnails import ThumbManager -from home.src.download.yt_dlp_base import CookieHandler from home.src.es.connect import ElasticWrap, IndexPaginate from home.src.ta.config import AppConfig from home.src.ta.settings import EnvironmentSettings diff --git a/tubearchivist/home/src/ta/urlparser.py b/tubearchivist/home/src/ta/urlparser.py index 662f51d4..15ed72f9 100644 --- a/tubearchivist/home/src/ta/urlparser.py +++ b/tubearchivist/home/src/ta/urlparser.py @@ -6,7 +6,7 @@ Functionality: from urllib.parse import parse_qs, urlparse -from home.src.download.yt_dlp_base import YtWrap +from download.src.yt_dlp_base import YtWrap from video.src.constants import VideoTypeEnum diff --git a/tubearchivist/home/tasks.py b/tubearchivist/home/tasks.py index 0fa8e726..79bce6ea 100644 --- a/tubearchivist/home/tasks.py +++ b/tubearchivist/home/tasks.py @@ -9,13 +9,10 @@ Functionality: from celery import Task, shared_task from celery.exceptions import Retry from channel.src.index import YoutubeChannel -from home.src.download.queue import PendingList -from home.src.download.subscriptions import ( - SubscriptionHandler, - SubscriptionScanner, -) -from home.src.download.thumbnails import ThumbFilesystem, ThumbValidator -from home.src.download.yt_dlp_handler import VideoDownloader +from download.src.queue import PendingList +from download.src.subscriptions import SubscriptionHandler, SubscriptionScanner +from download.src.thumbnails import ThumbFilesystem, ThumbValidator +from download.src.yt_dlp_handler import VideoDownloader from home.src.es.backup import ElasticBackup from home.src.es.index_setup import ElasitIndexWrap from home.src.index.filesystem import Scanner diff --git a/tubearchivist/home/views.py b/tubearchivist/home/views.py index 049b8d6e..f176a6a2 100644 --- a/tubearchivist/home/views.py +++ b/tubearchivist/home/views.py @@ -20,9 +20,9 @@ from django.http import Http404 from django.shortcuts import redirect, render from django.utils.decorators import method_decorator from django.views import View +from download.src.queue import PendingInteract +from download.src.yt_dlp_base import CookieHandler from home.models import CustomPeriodicTask -from home.src.download.queue import PendingInteract -from home.src.download.yt_dlp_base import CookieHandler from home.src.es.backup import ElasticBackup from home.src.es.connect import ElasticWrap from home.src.es.snapshot import ElasticSnapshot diff --git a/tubearchivist/playlist/src/index.py b/tubearchivist/playlist/src/index.py index 91806856..ffd316a6 100644 --- a/tubearchivist/playlist/src/index.py +++ b/tubearchivist/playlist/src/index.py @@ -8,7 +8,7 @@ import json from datetime import datetime from channel.src import index as channel -from home.src.download.thumbnails import ThumbManager +from download.src.thumbnails import ThumbManager from home.src.es.connect import ElasticWrap, IndexPaginate from home.src.index.generic import YouTubeItem from video.src.index import YoutubeVideo diff --git a/tubearchivist/playlist/views.py b/tubearchivist/playlist/views.py index 66e4edb8..40ae5205 100644 --- a/tubearchivist/playlist/views.py +++ b/tubearchivist/playlist/views.py @@ -1,7 +1,7 @@ """all playlist API views""" from api.views import AdminWriteOnly, ApiBaseView -from home.src.download.subscriptions import PlaylistSubscription +from download.src.subscriptions import PlaylistSubscription from home.src.ta.users import UserConfig from home.tasks import subscribe_to from playlist.src.index import YoutubePlaylist diff --git a/tubearchivist/video/src/comments.py b/tubearchivist/video/src/comments.py index 794cbc35..6723e9af 100644 --- a/tubearchivist/video/src/comments.py +++ b/tubearchivist/video/src/comments.py @@ -7,7 +7,7 @@ Functionality: from datetime import datetime -from home.src.download.yt_dlp_base import YtWrap +from download.src.yt_dlp_base import YtWrap from home.src.es.connect import ElasticWrap from home.src.ta.config import AppConfig from home.src.ta.ta_redis import RedisQueue