From 78d8bc9d249e9e024c223527d7c013d3a02827d5 Mon Sep 17 00:00:00 2001
From: simon <simobilleter@gmail.com>
Date: Thu, 23 Sep 2021 18:09:46 +0700
Subject: [PATCH] changed dl queue to redis, changed sort order to represent
 actual queue order

---
 tubearchivist/home/src/download.py | 40 +++++++++++++++++++++---------
 tubearchivist/home/views.py        |  2 +-
 2 files changed, 29 insertions(+), 13 deletions(-)

diff --git a/tubearchivist/home/src/download.py b/tubearchivist/home/src/download.py
index 4c61546..f915e79 100644
--- a/tubearchivist/home/src/download.py
+++ b/tubearchivist/home/src/download.py
@@ -14,7 +14,12 @@ from time import sleep
 import requests
 import yt_dlp as youtube_dl
 from home.src.config import AppConfig
-from home.src.helper import DurationConverter, clean_string, set_message
+from home.src.helper import (
+    DurationConverter,
+    RedisQueue,
+    clean_string,
+    set_message,
+)
 from home.src.index import YoutubeChannel, index_new_video
 
 
@@ -147,7 +152,7 @@ class PendingList:
             "size": 50,
             "query": {"match_all": {}},
             "pit": {"id": pit_id, "keep_alive": "1m"},
-            "sort": [{"timestamp": {"order": "desc"}}],
+            "sort": [{"timestamp": {"order": "asc"}}],
         }
         query_str = json.dumps(data)
         url = self.ES_URL + "/_search"
@@ -400,19 +405,24 @@ def playlist_extractor(playlist_id):
 
 
 class VideoDownloader:
-    """handle the video download functionality"""
+    """
+    handle the video download functionality
+    if not initiated with list, take from queue
+    """
 
-    def __init__(self, youtube_id_list):
+    def __init__(self, youtube_id_list=False):
         self.youtube_id_list = youtube_id_list
         self.config = AppConfig().config
 
-    def download_list(self):
-        """download the list of youtube_ids"""
-        limit_count = self.config["downloads"]["limit_count"]
-        if limit_count:
-            self.youtube_id_list = self.youtube_id_list[:limit_count]
+    def run_queue(self):
+        """setup download queue in redis loop until no more items"""
+        queue = RedisQueue("dl_queue")
+
+        while True:
+            youtube_id = queue.get_next()
+            if not youtube_id:
+                break
 
-        for youtube_id in self.youtube_id_list:
             try:
                 self.dl_single_vid(youtube_id)
             except youtube_dl.utils.DownloadError:
@@ -421,8 +431,14 @@ class VideoDownloader:
             vid_dict = index_new_video(youtube_id)
             self.move_to_archive(vid_dict)
             self.delete_from_pending(youtube_id)
-            if self.config["downloads"]["sleep_interval"]:
-                sleep(self.config["downloads"]["sleep_interval"])
+
+    @staticmethod
+    def add_pending():
+        """add pending videos to download queue"""
+        all_pending, _ = PendingList().get_all_pending()
+        to_add = [i["youtube_id"] for i in all_pending]
+        queue = RedisQueue("dl_queue")
+        queue.add_list(to_add)
 
     @staticmethod
     def progress_hook(response):
diff --git a/tubearchivist/home/views.py b/tubearchivist/home/views.py
index a64e0aa..95d0f9e 100644
--- a/tubearchivist/home/views.py
+++ b/tubearchivist/home/views.py
@@ -185,7 +185,7 @@ class DownloadView(View):
             "size": page_size,
             "from": page_from,
             "query": {"term": {"status": {"value": "pending"}}},
-            "sort": [{"timestamp": {"order": "desc"}}],
+            "sort": [{"timestamp": {"order": "asc"}}],
         }
         return data