tubearchivist/tubearchivist-frontend
1
0
Fork 0
mirror of https://github.com/tubearchivist/tubearchivist-frontend.git synced 2024-11-22 11:50:14 +00:00
Code Issues Projects Releases Wiki Activity

changed dl queue to redis, changed sort order to represent actual queue order

Browse Source
This commit is contained in:
simon 2021-09-23 18:09:46 +07:00
parent 7f57cabbc6
commit 78d8bc9d24
2 changed files with 29 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
tubearchivist/home/src/download.py
View File

@ -14,7 +14,12 @@ from time import sleep
import requests import requests
import yt_dlp as youtube_dl import yt_dlp as youtube_dl
from home.src.config import AppConfig from home.src.config import AppConfig
from home.src.helper import DurationConverter, clean_string, set_message from home.src.helper import (
DurationConverter,
RedisQueue,
clean_string,
set_message,
)
from home.src.index import YoutubeChannel, index_new_video from home.src.index import YoutubeChannel, index_new_video
@ -147,7 +152,7 @@ class PendingList:
"size": 50, "size": 50,
"query": {"match_all": {}}, "query": {"match_all": {}},
"pit": {"id": pit_id, "keep_alive": "1m"}, "pit": {"id": pit_id, "keep_alive": "1m"},
"sort": [{"timestamp": {"order": "desc"}}], "sort": [{"timestamp": {"order": "asc"}}],
} }
query_str = json.dumps(data) query_str = json.dumps(data)
url = self.ES_URL + "/_search" url = self.ES_URL + "/_search"
@ -400,19 +405,24 @@ def playlist_extractor(playlist_id):
class VideoDownloader: class VideoDownloader:
"""handle the video download functionality""" """
handle the video download functionality
if not initiated with list, take from queue
"""
def __init__(self, youtube_id_list): def __init__(self, youtube_id_list=False):
self.youtube_id_list = youtube_id_list self.youtube_id_list = youtube_id_list
self.config = AppConfig().config self.config = AppConfig().config
def download_list(self): def run_queue(self):
"""download the list of youtube_ids""" """setup download queue in redis loop until no more items"""
limit_count = self.config["downloads"]["limit_count"] queue = RedisQueue("dl_queue")
if limit_count:
self.youtube_id_list = self.youtube_id_list[:limit_count] while True:
youtube_id = queue.get_next()
if not youtube_id:
break
for youtube_id in self.youtube_id_list:
try: try:
self.dl_single_vid(youtube_id) self.dl_single_vid(youtube_id)
except youtube_dl.utils.DownloadError: except youtube_dl.utils.DownloadError:
@ -421,8 +431,14 @@ class VideoDownloader:
vid_dict = index_new_video(youtube_id) vid_dict = index_new_video(youtube_id)
self.move_to_archive(vid_dict) self.move_to_archive(vid_dict)
self.delete_from_pending(youtube_id) self.delete_from_pending(youtube_id)
if self.config["downloads"]["sleep_interval"]:
sleep(self.config["downloads"]["sleep_interval"]) @staticmethod
def add_pending():
"""add pending videos to download queue"""
all_pending, _ = PendingList().get_all_pending()
to_add = [i["youtube_id"] for i in all_pending]
queue = RedisQueue("dl_queue")
queue.add_list(to_add)
@staticmethod @staticmethod
def progress_hook(response): def progress_hook(response):

2
tubearchivist/home/views.py
View File

@ -185,7 +185,7 @@ class DownloadView(View):
"size": page_size, "size": page_size,
"from": page_from, "from": page_from,
"query": {"term": {"status": {"value": "pending"}}}, "query": {"term": {"status": {"value": "pending"}}},
"sort": [{"timestamp": {"order": "desc"}}], "sort": [{"timestamp": {"order": "asc"}}],
} }
return data return data