mirror of
https://github.com/tubearchivist/tubearchivist.git
synced 2024-12-13 05:30:12 +00:00
changed dl queue to redis, changed sort order to represent actual queue order
This commit is contained in:
parent
7f57cabbc6
commit
78d8bc9d24
@ -14,7 +14,12 @@ from time import sleep
|
||||
import requests
|
||||
import yt_dlp as youtube_dl
|
||||
from home.src.config import AppConfig
|
||||
from home.src.helper import DurationConverter, clean_string, set_message
|
||||
from home.src.helper import (
|
||||
DurationConverter,
|
||||
RedisQueue,
|
||||
clean_string,
|
||||
set_message,
|
||||
)
|
||||
from home.src.index import YoutubeChannel, index_new_video
|
||||
|
||||
|
||||
@ -147,7 +152,7 @@ class PendingList:
|
||||
"size": 50,
|
||||
"query": {"match_all": {}},
|
||||
"pit": {"id": pit_id, "keep_alive": "1m"},
|
||||
"sort": [{"timestamp": {"order": "desc"}}],
|
||||
"sort": [{"timestamp": {"order": "asc"}}],
|
||||
}
|
||||
query_str = json.dumps(data)
|
||||
url = self.ES_URL + "/_search"
|
||||
@ -400,19 +405,24 @@ def playlist_extractor(playlist_id):
|
||||
|
||||
|
||||
class VideoDownloader:
|
||||
"""handle the video download functionality"""
|
||||
"""
|
||||
handle the video download functionality
|
||||
if not initiated with list, take from queue
|
||||
"""
|
||||
|
||||
def __init__(self, youtube_id_list):
|
||||
def __init__(self, youtube_id_list=False):
|
||||
self.youtube_id_list = youtube_id_list
|
||||
self.config = AppConfig().config
|
||||
|
||||
def download_list(self):
|
||||
"""download the list of youtube_ids"""
|
||||
limit_count = self.config["downloads"]["limit_count"]
|
||||
if limit_count:
|
||||
self.youtube_id_list = self.youtube_id_list[:limit_count]
|
||||
def run_queue(self):
|
||||
"""setup download queue in redis loop until no more items"""
|
||||
queue = RedisQueue("dl_queue")
|
||||
|
||||
while True:
|
||||
youtube_id = queue.get_next()
|
||||
if not youtube_id:
|
||||
break
|
||||
|
||||
for youtube_id in self.youtube_id_list:
|
||||
try:
|
||||
self.dl_single_vid(youtube_id)
|
||||
except youtube_dl.utils.DownloadError:
|
||||
@ -421,8 +431,14 @@ class VideoDownloader:
|
||||
vid_dict = index_new_video(youtube_id)
|
||||
self.move_to_archive(vid_dict)
|
||||
self.delete_from_pending(youtube_id)
|
||||
if self.config["downloads"]["sleep_interval"]:
|
||||
sleep(self.config["downloads"]["sleep_interval"])
|
||||
|
||||
@staticmethod
|
||||
def add_pending():
|
||||
"""add pending videos to download queue"""
|
||||
all_pending, _ = PendingList().get_all_pending()
|
||||
to_add = [i["youtube_id"] for i in all_pending]
|
||||
queue = RedisQueue("dl_queue")
|
||||
queue.add_list(to_add)
|
||||
|
||||
@staticmethod
|
||||
def progress_hook(response):
|
||||
|
@ -185,7 +185,7 @@ class DownloadView(View):
|
||||
"size": page_size,
|
||||
"from": page_from,
|
||||
"query": {"term": {"status": {"value": "pending"}}},
|
||||
"sort": [{"timestamp": {"order": "desc"}}],
|
||||
"sort": [{"timestamp": {"order": "asc"}}],
|
||||
}
|
||||
return data
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user