tubearchivist
/
tubearchivist
mirror of https://github.com/tubearchivist/tubearchivist.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

use the refactored PendingList class

This commit is contained in:
simon 2022-03-18 18:27:25 +07:00
parent 05b8dbc02f
commit a6937db5fd
No known key found for this signature in database
GPG Key ID: 2C15AA5E89985DD4
8 changed files with 75 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
tubearchivist/home/src/download/queue.py
View File

@ -106,7 +106,7 @@ class PendingInteract:
class PendingList(PendingIndex):
"""manage the pending videos list"""
def __init__(self, youtube_ids):
def __init__(self, youtube_ids=False):
super().__init__()
self.youtube_ids = youtube_ids
self.to_skip = False
@ -173,7 +173,7 @@ class PendingList(PendingIndex):
thumb_handler = ThumbManager()
for idx, youtube_id in enumerate(self.missing_videos):
video_details = self._get_youtube_details(youtube_id)
video_details = self.get_youtube_details(youtube_id)
if not video_details:
continue
@ -208,7 +208,7 @@ class PendingList(PendingIndex):
if idx + 1 % 25 == 0:
print("adding to queue progress: " + progress)
def _get_youtube_details(self, youtube_id):
def get_youtube_details(self, youtube_id):
"""get details from youtubedl for single pending video"""
obs = {
"default_search": "ytsearch",

27
tubearchivist/home/src/download/subscriptions.py
View File

@ -61,11 +61,9 @@ class ChannelSubscription:
def find_missing(self):
"""add missing videos from subscribed channels to pending"""
all_channels = self.get_channels()
pending_handler = queue.PendingList()
all_pending, all_ignore = pending_handler.get_all_pending()
all_ids = [i["youtube_id"] for i in all_ignore + all_pending]
all_downloaded = pending_handler.get_all_downloaded()
to_ignore = all_ids + all_downloaded
pending = queue.PendingList()
pending.get_download()
pending.get_indexed()
missing_videos = []
@ -75,7 +73,7 @@ class ChannelSubscription:
if last_videos:
for video in last_videos:
if video[0] not in to_ignore:
if video[0] not in pending.to_skip:
missing_videos.append(video[0])
# notify
message = {
@ -129,7 +127,11 @@ class PlaylistSubscription:
def process_url_str(self, new_playlists, subscribed=True):
"""process playlist subscribe form url_str"""
all_indexed = queue.PendingList().get_all_indexed()
data = {
"query": {"match_all": {}},
"sort": [{"published": {"order": "desc"}}],
}
all_indexed = IndexPaginate("ta_video", data).get_results()
all_youtube_ids = [i["youtube_id"] for i in all_indexed]
new_thumbs = []
@ -179,12 +181,11 @@ class PlaylistSubscription:
@staticmethod
def get_to_ignore():
"""get all youtube_ids already downloaded or ignored"""
pending_handler = queue.PendingList()
all_pending, all_ignore = pending_handler.get_all_pending()
all_ids = [i["youtube_id"] for i in all_ignore + all_pending]
all_downloaded = pending_handler.get_all_downloaded()
to_ignore = all_ids + all_downloaded
return to_ignore
pending = queue.PendingList()
pending.get_download()
pending.get_indexed()
return pending.to_skip
def find_missing(self):
"""find videos in subscribed playlists not downloaded yet"""

17
tubearchivist/home/src/download/thumbnails.py
View File

@ -58,11 +58,12 @@ class ThumbManager:
def get_needed_thumbs(self, missing_only=False):
"""get a list of all missing thumbnails"""
all_thumbs = self.get_all_thumbs()
all_indexed = queue.PendingList().get_all_indexed()
all_in_queue, all_ignored = queue.PendingList().get_all_pending()
pending = queue.PendingList()
pending.get_indexed()
needed_thumbs = []
for video in all_indexed:
for video in pending.all_videos:
youtube_id = video["youtube_id"]
thumb_url = video["vid_thumb_url"]
if missing_only:
@ -71,7 +72,9 @@ class ThumbManager:
else:
needed_thumbs.append((youtube_id, thumb_url))
for video in all_in_queue + all_ignored:
pending.get_download()
for video in pending.all_pending + pending.all_ignored:
youtube_id = video["youtube_id"]
thumb_url = video["vid_thumb_url"]
if missing_only:
@ -276,9 +279,11 @@ class ThumbManager:
def get_thumb_list(self):
"""get list of mediafiles and matching thumbnails"""
all_indexed = queue.PendingList().get_all_indexed()
pending = queue.PendingList()
pending.get_indexed()
video_list = []
for video in all_indexed:
for video in pending.all_videos:
youtube_id = video["youtube_id"]
media_url = os.path.join(self.MEDIA_DIR, video["media_url"])
thumb_path = os.path.join(

24
tubearchivist/home/src/download/yt_dlp_handler.py
View File

@ -75,8 +75,9 @@ class VideoDownloader:
"message": "Scanning your download queue.",
}
RedisArchivist().set_message("message:download", mess_dict)
all_pending, _ = PendingList().get_all_pending()
to_add = [i["youtube_id"] for i in all_pending]
pending = PendingList()
pending.get_download()
to_add = [i["youtube_id"] for i in pending.all_pending]
if not to_add:
# there is nothing pending
print("download queue is empty")
@ -259,8 +260,9 @@ class VideoDownloader:
"""look for playlist needing to update"""
print("sync playlists")
self._add_subscribed_channels()
all_indexed = PendingList().get_all_indexed()
all_youtube_ids = [i["youtube_id"] for i in all_indexed]
pending = PendingList()
pending.get_indexed()
all_youtube_ids = [i["youtube_id"] for i in pending.all_videos]
for id_c, channel_id in enumerate(self.channels):
playlists = YoutubeChannel(channel_id).get_indexed_playlists()
all_playlist_ids = [i["playlist_id"] for i in playlists]
@ -302,15 +304,17 @@ class VideoDownloader:
"query": {"range": {"player.watched_date": {"lte": now_lte}}},
"sort": [{"player.watched_date": {"order": "asc"}}],
}
all_to_delete = IndexPaginate("ta_video", data).get_results()
all_youtube_ids = [i["youtube_id"] for i in all_to_delete]
if not all_youtube_ids:
to_delete = IndexPaginate("ta_video", data).get_results()
if not to_delete:
return
for youtube_id in all_youtube_ids:
for video in to_delete:
youtube_id = video["youtube_id"]
print(f"autodelete {youtube_id}")
YoutubeVideo(youtube_id).delete_media_file()
print("add deleted to ignore list")
pending_handler = PendingList()
pending_handler.add_to_pending(all_youtube_ids, ignore=True)
vids = [{"type": "video", "url": i["youtube_id"]} for i in to_delete]
pending = PendingList(youtube_ids=vids)
pending.parse_url_list()
pending.add_to_pending(status="ignore")

32
tubearchivist/home/src/frontend/api_calls.py
View File

@ -4,7 +4,7 @@ Functionality:
- called via user input
"""
from home.src.download.queue import PendingList
from home.src.download.queue import PendingInteract
from home.src.download.subscriptions import (
ChannelSubscription,
PlaylistSubscription,
@ -110,12 +110,11 @@ class PostData:
def _ignore(self):
"""ignore from download queue"""
id_to_ignore = self.exec_val
print("ignore video " + id_to_ignore)
handler = PendingList()
handler.ignore_from_pending([id_to_ignore])
video_id = self.exec_val
print(f"ignore video {video_id}")
PendingInteract(video_id=video_id, status="ignore")
# also clear from redis queue
RedisQueue("dl_queue").clear_item(id_to_ignore)
RedisQueue("dl_queue").clear_item(video_id)
return {"success": True}
@staticmethod
@ -222,28 +221,25 @@ class PostData:
def _forget_ignore(self):
"""delete from ta_download index"""
youtube_id = self.exec_val
print("forgetting from download index: " + youtube_id)
PendingList().delete_from_pending(youtube_id)
video_id = self.exec_val
print(f"forgetting from download index: {video_id}")
PendingInteract(video_id=video_id).delete_item()
return {"success": True}
def _add_single(self):
"""add single youtube_id to download queue"""
youtube_id = self.exec_val
print("add vid to dl queue: " + youtube_id)
PendingList().delete_from_pending(youtube_id)
youtube_ids = UrlListParser(youtube_id).process_list()
extrac_dl.delay(youtube_ids)
video_id = self.exec_val
print(f"add vid to dl queue: {video_id}")
PendingInteract(video_id=video_id).delete_item()
video_ids = UrlListParser(video_id).process_list()
extrac_dl.delay(video_ids)
return {"success": True}
def _delete_queue(self):
"""delete download queue"""
status = self.exec_val
print("deleting from download queue: " + status)
if status == "pending":
PendingList().delete_pending("pending")
elif status == "ignore":
PendingList().delete_pending("ignore")
PendingInteract(status=status).delete_by_status()
return {"success": True}
@staticmethod

5
tubearchivist/home/src/index/filesystem.py
View File

@ -59,9 +59,10 @@ class FilesystemScanner:
def get_all_indexed():
"""get a list of all indexed videos"""
index_handler = PendingList()
all_indexed_raw = index_handler.get_all_indexed()
index_handler.get_indexed()
all_indexed = []
for video in all_indexed_raw:
for video in index_handler.all_videos:
youtube_id = video["youtube_id"]
media_url = video["media_url"]
published = video["published"]

5
tubearchivist/home/src/index/reindex.py
View File

@ -266,8 +266,9 @@ class Reindex:
# playlist
print(f"reindexing {len(self.all_playlist_ids)} playlists")
if self.all_playlist_ids:
all_indexed = PendingList().get_all_indexed()
all_indexed_ids = [i["youtube_id"] for i in all_indexed]
handler = PendingList()
handler.get_indexed()
all_indexed_ids = [i["youtube_id"] for i in handler.all_videos]
for playlist_id in self.all_playlist_ids:
self.reindex_single_playlist(playlist_id, all_indexed_ids)
if self.sleep_interval:

31
tubearchivist/home/tasks.py
View File

@ -63,9 +63,13 @@ def update_subscribed():
missing_from_playlists = playlist_handler.find_missing()
missing = missing_from_channels + missing_from_playlists
if missing:
pending_handler = PendingList()
all_videos_added = pending_handler.add_to_pending(missing)
ThumbManager().download_vid(all_videos_added)
youtube_ids = [
{"type": "video", "url": i["youtube_id"]} for i in missing
]
pending_handler = PendingList(youtube_ids=youtube_ids)
pending_handler.parse_url_list()
pending_handler.add_to_pending()
else:
print("Did not acquire rescan lock.")
@ -128,19 +132,9 @@ def download_single(youtube_id):
@shared_task
def extrac_dl(youtube_ids):
"""parse list passed and add to pending"""
pending_handler = PendingList()
missing_videos = pending_handler.parse_url_list(youtube_ids)
all_videos_added = pending_handler.add_to_pending(missing_videos)
missing_playlists = pending_handler.missing_from_playlists
thumb_handler = ThumbManager()
if missing_playlists:
new_thumbs = PlaylistSubscription().process_url_str(
missing_playlists, subscribed=False
)
thumb_handler.download_playlist(new_thumbs)
thumb_handler.download_vid(all_videos_added)
pending_handler = PendingList(youtube_ids=youtube_ids)
pending_handler.parse_url_list()
pending_handler.add_to_pending()
@shared_task(name="check_reindex")
@ -283,8 +277,9 @@ def index_channel_playlists(channel_id):
print(f"no playlists found for channel {channel_id}")
return
all_indexed = PendingList().get_all_indexed()
all_youtube_ids = [i["youtube_id"] for i in all_indexed]
handler = PendingList()
handler.get_indexed()
all_youtube_ids = [i["youtube_id"] for i in handler.all_videos]
for idx, (playlist_id, playlist_title) in enumerate(all_playlists):
# notify