From 60f1809ed80e8b56920c554bc15017654fd04d19 Mon Sep 17 00:00:00 2001 From: simon Date: Wed, 17 May 2023 23:24:47 +0700 Subject: [PATCH 01/38] fix rescan without task --- tubearchivist/home/src/index/reindex.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tubearchivist/home/src/index/reindex.py b/tubearchivist/home/src/index/reindex.py index 7473c89..b5c8459 100644 --- a/tubearchivist/home/src/index/reindex.py +++ b/tubearchivist/home/src/index/reindex.py @@ -249,7 +249,8 @@ class Reindex(ReindexBase): reindex = self.get_reindex_map(index_config["index_name"]) youtube_id = RedisQueue(index_config["queue_name"]).get_next() if youtube_id: - self._notify(name, index_config, total) + if self.task: + self._notify(name, index_config, total) reindex(youtube_id) sleep_interval = self.config["downloads"].get("sleep_interval", 0) sleep(sleep_interval) From 918a04c5022865bda795c46553bb783733b4b640 Mon Sep 17 00:00:00 2001 From: simon Date: Thu, 18 May 2023 17:01:07 +0700 Subject: [PATCH 02/38] allow empty data for paginate --- tubearchivist/home/src/es/connect.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tubearchivist/home/src/es/connect.py b/tubearchivist/home/src/es/connect.py index 5af1838..134472c 100644 --- a/tubearchivist/home/src/es/connect.py +++ b/tubearchivist/home/src/es/connect.py @@ -127,6 +127,9 @@ class IndexPaginate: def validate_data(self): """add pit and size to data""" + if not self.data: + self.data = {} + if "sort" not in self.data.keys(): self.data.update({"sort": [{"_doc": {"order": "desc"}}]}) From d62b0d3f8dfd86171d6f17b9cbefddb057a80895 Mon Sep 17 00:00:00 2001 From: simon Date: Thu, 18 May 2023 17:42:15 +0700 Subject: [PATCH 03/38] implement simplified path migration --- .../config/management/commands/ta_migpath.py | 182 ++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100644 tubearchivist/config/management/commands/ta_migpath.py diff --git a/tubearchivist/config/management/commands/ta_migpath.py b/tubearchivist/config/management/commands/ta_migpath.py new file mode 100644 index 0000000..6137f5b --- /dev/null +++ b/tubearchivist/config/management/commands/ta_migpath.py @@ -0,0 +1,182 @@ +"""filepath migration from v0.3.6 to v0.3.7""" + +import json +import os + +from django.core.management.base import BaseCommand +from home.src.es.connect import ElasticWrap, IndexPaginate +from home.src.ta.config import AppConfig +from home.src.ta.helper import ignore_filelist + +TOPIC = """ + +######################## +# Filesystem Migration # +######################## + +""" + + +class Command(BaseCommand): + """command framework""" + + # pylint: disable=no-member + + def handle(self, *args, **options): + """run commands""" + self.stdout.write(TOPIC) + need_migration = self.channels_need_migration() + if not need_migration: + self.stdout.write( + self.style.SUCCESS(" no channel migration needed") + ) + return + + self.stdout.write( + self.style.SUCCESS(f" migrating {len(need_migration)} channels") + ) + for channel in need_migration: + channel_name = channel["channel_name"] + channel_id = channel["channel_id"] + self.stdout.write( + self.style.SUCCESS( + f" migrating {channel_name} [{channel_id}]" + ) + ) + ChannelMigration(channel).migrate() + + self.stdout.write( + self.style.SUCCESS(" ✓ channel migration completed") + ) + + def channels_need_migration(self): + """get channels that need migration""" + all_indexed = self._get_channel_indexed() + all_folders = self._get_channel_folders() + need_migration = [] + for channel in all_indexed: + if channel["channel_id"] not in all_folders: + need_migration.append(channel) + + return need_migration + + def _get_channel_indexed(self): + """get all channels indexed""" + all_results = IndexPaginate("ta_channel", False).get_results() + + return all_results + + def _get_channel_folders(self): + """get all channel folders""" + base_folder = AppConfig().config["application"]["videos"] + existing_folders = ignore_filelist(os.listdir(base_folder)) + + return existing_folders + + +class ChannelMigration: + """migrate single channel""" + + def __init__(self, channel): + self.channel = channel + self.videos = AppConfig().config["application"]["videos"] + self.bulk_list = [] + + def migrate(self): + """run migration""" + self._create_new_folder() + all_videos = self.get_channel_videos() + self.migrate_videos(all_videos) + self.send_bulk() + self.delete_old(all_videos) + + def _create_new_folder(self): + """create new channel id folder""" + new_path = os.path.join(self.videos, self.channel["channel_id"]) + if not os.path.exists(new_path): + os.mkdir(new_path) + + def get_channel_videos(self): + """get all videos of channel""" + data = { + "query": { + "term": { + "channel.channel_id": {"value": self.channel["channel_id"]} + } + } + } + all_videos = IndexPaginate("ta_video", data).get_results() + + return all_videos + + def migrate_videos(self, all_videos): + """migrate all videos of channel""" + for video in all_videos: + new_media_url = self._move_video_file(video) + all_subtitles = self._move_subtitles(video) + action = { + "update": {"_id": video["youtube_id"], "_index": "ta_video"} + } + source = {"doc": {"media_url": new_media_url}} + if all_subtitles: + source["doc"].update({"subtitles": all_subtitles}) + + self.bulk_list.append(json.dumps(action)) + self.bulk_list.append(json.dumps(source)) + + def _move_video_file(self, video): + """move video file to new location""" + old_path = os.path.join(self.videos, video["media_url"]) + if not os.path.exists(old_path): + print(f"did not find expected video at {old_path}") + return False + + new_media_url = os.path.join( + self.channel["channel_id"], video["youtube_id"] + ".mp4" + ) + os.rename(old_path, os.path.join(self.videos, new_media_url)) + + return new_media_url + + def _move_subtitles(self, video): + """move subtitle files to new location""" + all_subtitles = video.get("subtitles") + if not all_subtitles: + return False + + for subtitle in all_subtitles: + old_path = os.path.join(self.videos, subtitle["media_url"]) + if not os.path.exists(old_path): + print(f"did not find expected subtitle at {old_path}") + continue + + ext = ".".join(old_path.split(".")[-2:]) + new_media_url = os.path.join( + self.channel["channel_id"], video["youtube_id"] + f".{ext}" + ) + os.rename(old_path, os.path.join(self.videos, new_media_url)) + subtitle["media_url"] = new_media_url + + return all_subtitles + + def send_bulk(self): + """send bulk request to update index with new urls""" + if not self.bulk_list: + print("nothing to update") + return + + self.bulk_list.append("\n") + data = "\n".join(self.bulk_list) + response, status = ElasticWrap("_bulk").post(data=data, ndjson=True) + if not status == 200: + print(response) + + def delete_old(self, all_videos): + """delete old folder path if empty""" + channel_name = os.path.split(all_videos[0]["media_url"])[0] + old_path = os.path.join(self.videos, channel_name) + if os.path.exists(old_path) and not os.listdir(old_path): + os.rmdir(old_path) + return + + print(f"failed to clean up old folder {old_path}") From 9d6ab6b7b3c5d334cf167365a84e8de7608c37f1 Mon Sep 17 00:00:00 2001 From: simon Date: Thu, 18 May 2023 20:32:37 +0700 Subject: [PATCH 04/38] implement new media_url --- .../home/src/download/yt_dlp_handler.py | 27 +++++++------------ tubearchivist/home/src/index/video.py | 19 ++++--------- 2 files changed, 15 insertions(+), 31 deletions(-) diff --git a/tubearchivist/home/src/download/yt_dlp_handler.py b/tubearchivist/home/src/download/yt_dlp_handler.py index 700eee5..0ea813a 100644 --- a/tubearchivist/home/src/download/yt_dlp_handler.py +++ b/tubearchivist/home/src/download/yt_dlp_handler.py @@ -20,7 +20,7 @@ from home.src.index.playlist import YoutubePlaylist from home.src.index.video import YoutubeVideo, index_new_video from home.src.index.video_constants import VideoTypeEnum from home.src.ta.config import AppConfig -from home.src.ta.helper import clean_string, ignore_filelist +from home.src.ta.helper import ignore_filelist class DownloadPostProcess: @@ -361,23 +361,16 @@ class VideoDownloader: videos = self.config["application"]["videos"] host_uid = self.config["application"]["HOST_UID"] host_gid = self.config["application"]["HOST_GID"] - channel_name = clean_string(vid_dict["channel"]["channel_name"]) - if len(channel_name) <= 3: - # fall back to channel id - channel_name = vid_dict["channel"]["channel_id"] - # make archive folder with correct permissions - new_folder = os.path.join(videos, channel_name) - if not os.path.exists(new_folder): - os.makedirs(new_folder) - if host_uid and host_gid: - os.chown(new_folder, host_uid, host_gid) - # find real filename + # make folder + folder = os.path.join(videos, vid_dict["channel"]["channel_id"]) + if not os.path.exists(folder): + os.makedirs(folder) + if host_uid and host_gid: + os.chown(folder, host_uid, host_gid) + # move media file + media_file = vid_dict["youtube_id"] + ".mp4" cache_dir = self.config["application"]["cache_dir"] - all_cached = ignore_filelist(os.listdir(cache_dir + "/download/")) - for file_str in all_cached: - if vid_dict["youtube_id"] in file_str: - old_file = file_str - old_path = os.path.join(cache_dir, "download", old_file) + old_path = os.path.join(cache_dir, "download", media_file) new_path = os.path.join(videos, vid_dict["media_url"]) # move media file and fix permission shutil.move(old_path, new_path, copy_function=shutil.copyfile) diff --git a/tubearchivist/home/src/index/video.py b/tubearchivist/home/src/index/video.py index b4daae2..7b2da2d 100644 --- a/tubearchivist/home/src/index/video.py +++ b/tubearchivist/home/src/index/video.py @@ -20,7 +20,7 @@ from home.src.index.video_streams import ( DurationConverter, MediaStreamExtractor, ) -from home.src.ta.helper import clean_string, randomizor +from home.src.ta.helper import randomizor from home.src.ta.ta_redis import RedisArchivist from ryd_client import ryd_client @@ -292,19 +292,10 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): def add_file_path(self): """build media_url for where file will be located""" - channel_name = self.json_data["channel"]["channel_name"] - clean_channel_name = clean_string(channel_name) - if len(clean_channel_name) <= 3: - # fall back to channel id - clean_channel_name = self.json_data["channel"]["channel_id"] - - timestamp = self.json_data["published"].replace("-", "") - youtube_id = self.json_data["youtube_id"] - title = self.json_data["title"] - clean_title = clean_string(title) - filename = f"{timestamp}_{youtube_id}_{clean_title}.mp4" - media_url = os.path.join(clean_channel_name, filename) - self.json_data["media_url"] = media_url + self.json_data["media_url"] = os.path.join( + self.json_data["channel"]["channel_id"], + self.json_data["youtube_id"] + ".mp4", + ) def delete_media_file(self): """delete video file, meta data""" From 8ef59f5bff005e226c043c7853c5902dbf3d5e7c Mon Sep 17 00:00:00 2001 From: simon Date: Thu, 18 May 2023 20:32:58 +0700 Subject: [PATCH 05/38] delete channel path building --- tubearchivist/home/src/index/channel.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/tubearchivist/home/src/index/channel.py b/tubearchivist/home/src/index/channel.py index 7ceeccb..4203c3d 100644 --- a/tubearchivist/home/src/index/channel.py +++ b/tubearchivist/home/src/index/channel.py @@ -14,7 +14,6 @@ from home.src.download.yt_dlp_base import YtWrap from home.src.es.connect import ElasticWrap, IndexPaginate from home.src.index.generic import YouTubeItem from home.src.index.playlist import YoutubePlaylist -from home.src.ta.helper import clean_string class YoutubeChannel(YouTubeItem): @@ -177,12 +176,10 @@ class YoutubeChannel(YouTubeItem): def get_folder_path(self): """get folder where media files get stored""" - channel_name = self.json_data["channel_name"] - folder_name = clean_string(channel_name) - if len(folder_name) <= 3: - # fall back to channel id - folder_name = self.json_data["channel_id"] - folder_path = os.path.join(self.app_conf["videos"], folder_name) + folder_path = os.path.join( + self.app_conf["videos"], + self.json_data["channel_id"], + ) return folder_path def delete_es_videos(self): From 64984bc1b3aedf7a40fcaaaf3f7d68e1b96e2f67 Mon Sep 17 00:00:00 2001 From: simon Date: Fri, 19 May 2023 14:49:49 +0700 Subject: [PATCH 06/38] fix chown for mig new folder --- tubearchivist/config/management/commands/ta_migpath.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tubearchivist/config/management/commands/ta_migpath.py b/tubearchivist/config/management/commands/ta_migpath.py index 6137f5b..4d039a5 100644 --- a/tubearchivist/config/management/commands/ta_migpath.py +++ b/tubearchivist/config/management/commands/ta_migpath.py @@ -79,7 +79,8 @@ class ChannelMigration: def __init__(self, channel): self.channel = channel - self.videos = AppConfig().config["application"]["videos"] + self.config = AppConfig().config + self.videos = self.config["application"]["videos"] self.bulk_list = [] def migrate(self): @@ -92,9 +93,13 @@ class ChannelMigration: def _create_new_folder(self): """create new channel id folder""" + host_uid = self.config["application"]["HOST_UID"] + host_gid = self.config["application"]["HOST_GID"] new_path = os.path.join(self.videos, self.channel["channel_id"]) if not os.path.exists(new_path): os.mkdir(new_path) + if host_uid and host_gid: + os.chown(new_path, host_uid, host_gid) def get_channel_videos(self): """get all videos of channel""" From 5334d79d0d241ef26ea6cfa44fd3d7808981db14 Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 20 May 2023 15:38:55 +0700 Subject: [PATCH 07/38] default query --- tubearchivist/home/src/es/connect.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tubearchivist/home/src/es/connect.py b/tubearchivist/home/src/es/connect.py index 134472c..0b9d554 100644 --- a/tubearchivist/home/src/es/connect.py +++ b/tubearchivist/home/src/es/connect.py @@ -130,6 +130,9 @@ class IndexPaginate: if not self.data: self.data = {} + if "query" not in self.data.keys(): + self.data.update({"query": {"match_all": {}}}) + if "sort" not in self.data.keys(): self.data.update({"sort": [{"_doc": {"order": "desc"}}]}) From 9b30c7df6eaa20186eaf18e6baed7b1837ec96b3 Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 20 May 2023 16:07:33 +0700 Subject: [PATCH 08/38] refacter filesystem scanner --- tubearchivist/home/src/index/filesystem.py | 251 +++++++-------------- tubearchivist/home/tasks.py | 6 +- 2 files changed, 89 insertions(+), 168 deletions(-) diff --git a/tubearchivist/home/src/index/filesystem.py b/tubearchivist/home/src/index/filesystem.py index 75f4724..7594ee0 100644 --- a/tubearchivist/home/src/index/filesystem.py +++ b/tubearchivist/home/src/index/filesystem.py @@ -1,198 +1,85 @@ """ Functionality: -- reindexing old documents -- syncing updated values between indexes - scan the filesystem to delete or index """ -import json import os -from home.src.download.queue import PendingList -from home.src.es.connect import ElasticWrap +from home.src.es.connect import ElasticWrap, IndexPaginate from home.src.index.comments import CommentList -from home.src.index.video import index_new_video +from home.src.index.video import YoutubeVideo, index_new_video from home.src.ta.config import AppConfig -from home.src.ta.helper import clean_string, ignore_filelist -from PIL import ImageFile - -ImageFile.LOAD_TRUNCATED_IMAGES = True +from home.src.ta.helper import ignore_filelist -class ScannerBase: - """scan the filesystem base class""" +class Scanner: + """scan index and filesystem""" - CONFIG = AppConfig().config - VIDEOS = CONFIG["application"]["videos"] - - def __init__(self): - self.to_index = False - self.to_delete = False - self.mismatch = False - self.to_rename = False - - def scan(self): - """entry point, scan and compare""" - all_downloaded = self._get_all_downloaded() - all_indexed = self._get_all_indexed() - self.list_comarison(all_downloaded, all_indexed) - - def _get_all_downloaded(self): - """get a list of all video files downloaded""" - channels = os.listdir(self.VIDEOS) - all_channels = ignore_filelist(channels) - all_channels.sort() - all_downloaded = [] - for channel_name in all_channels: - channel_path = os.path.join(self.VIDEOS, channel_name) - channel_files = os.listdir(channel_path) - channel_files_clean = ignore_filelist(channel_files) - all_videos = [i for i in channel_files_clean if i.endswith(".mp4")] - for video in all_videos: - youtube_id = video[9:20] - all_downloaded.append((channel_name, video, youtube_id)) - - return all_downloaded - - @staticmethod - def _get_all_indexed(): - """get a list of all indexed videos""" - index_handler = PendingList() - index_handler.get_download() - index_handler.get_indexed() - - all_indexed = [] - for video in index_handler.all_videos: - youtube_id = video["youtube_id"] - media_url = video["media_url"] - published = video["published"] - title = video["title"] - all_indexed.append((youtube_id, media_url, published, title)) - return all_indexed - - def list_comarison(self, all_downloaded, all_indexed): - """compare the lists to figure out what to do""" - self._find_unindexed(all_downloaded, all_indexed) - self._find_missing(all_downloaded, all_indexed) - self._find_bad_media_url(all_downloaded, all_indexed) - - def _find_unindexed(self, all_downloaded, all_indexed): - """find video files without a matching document indexed""" - all_indexed_ids = [i[0] for i in all_indexed] - self.to_index = [] - for downloaded in all_downloaded: - if downloaded[2] not in all_indexed_ids: - self.to_index.append(downloaded) - - def _find_missing(self, all_downloaded, all_indexed): - """find indexed videos without matching media file""" - all_downloaded_ids = [i[2] for i in all_downloaded] - self.to_delete = [] - for video in all_indexed: - youtube_id = video[0] - if youtube_id not in all_downloaded_ids: - self.to_delete.append(video) - - def _find_bad_media_url(self, all_downloaded, all_indexed): - """rename media files not matching the indexed title""" - self.mismatch = [] - self.to_rename = [] - - for downloaded in all_downloaded: - channel, filename, downloaded_id = downloaded - # find in indexed - for indexed in all_indexed: - indexed_id, media_url, published, title = indexed - if indexed_id == downloaded_id: - # found it - pub = published.replace("-", "") - expected = f"{pub}_{indexed_id}_{clean_string(title)}.mp4" - new_url = os.path.join(channel, expected) - if expected != filename: - # file to rename - self.to_rename.append((channel, filename, expected)) - if media_url != new_url: - # media_url to update in es - self.mismatch.append((indexed_id, new_url)) - - break - - -class Filesystem(ScannerBase): - """handle scanning and fixing from filesystem""" + VIDEOS = AppConfig().config["application"]["videos"] def __init__(self, task=False): - super().__init__() self.task = task + self.to_delete = False + self.to_index = False - def process(self): - """entry point""" + def scan(self): + """scan the filesystem""" + downloaded = self._get_downloaded() + indexed = self._get_indexed() + self.to_index = downloaded - indexed + self.to_delete = indexed - downloaded + + def _get_downloaded(self): + """get downloaded ids""" if self.task: - self.task.send_progress(["Scanning your archive and index."]) - self.scan() - self.rename_files() - self.send_mismatch_bulk() - self.delete_from_index() - self.add_missing() + self.task.send_progress(["Scan your filesystem for videos."]) - def rename_files(self): - """rename media files as identified by find_bad_media_url""" - if not self.to_rename: - return + downloaded = set() + channels = ignore_filelist(os.listdir(self.VIDEOS)) + for channel in channels: + folder = os.path.join(self.VIDEOS, channel) + files = ignore_filelist(os.listdir(folder)) + downloaded.update(set(i.split(".")[0] for i in files)) - total = len(self.to_rename) + return downloaded + + def _get_indexed(self): + """get all indexed ids""" if self.task: - self.task.send_progress([f"Rename {total} media files."]) - for bad_filename in self.to_rename: - channel, filename, expected_filename = bad_filename - print(f"renaming [{filename}] to [{expected_filename}]") - old_path = os.path.join(self.VIDEOS, channel, filename) - new_path = os.path.join(self.VIDEOS, channel, expected_filename) - os.rename(old_path, new_path) + self.task.send_progress(["Get all videos indexed."]) - def send_mismatch_bulk(self): - """build bulk update""" - if not self.mismatch: - return + data = {"query": {"match_all": {}}, "_source": ["youtube_id"]} + response = IndexPaginate("ta_video", data).get_results() + return set(i["youtube_id"] for i in response) - total = len(self.mismatch) - if self.task: - self.task.send_progress([f"Fix media urls for {total} files"]) - bulk_list = [] - for video_mismatch in self.mismatch: - youtube_id, media_url = video_mismatch - print(f"{youtube_id}: fixing media url {media_url}") - action = {"update": {"_id": youtube_id, "_index": "ta_video"}} - source = {"doc": {"media_url": media_url}} - bulk_list.append(json.dumps(action)) - bulk_list.append(json.dumps(source)) - # add last newline - bulk_list.append("\n") - data = "\n".join(bulk_list) - _, _ = ElasticWrap("_bulk").post(data=data, ndjson=True) + def apply(self): + """apply all changes""" + self.delete() + self.index() + self.url_fix() - def delete_from_index(self): - """find indexed but deleted mediafile""" + def delete(self): + """delete videos from index""" if not self.to_delete: + print("nothing to delete") return - total = len(self.to_delete) if self.task: - self.task.send_progress([f"Clean up {total} items from index."]) - for indexed in self.to_delete: - youtube_id = indexed[0] - print(f"deleting {youtube_id} from index") - path = f"ta_video/_doc/{youtube_id}" - _, _ = ElasticWrap(path).delete() + self.task.send_progress( + [f"Remove {len(self.to_delete)} videos from index."] + ) - def add_missing(self): - """add missing videos to index""" - video_ids = [i[2] for i in self.to_index] - if not video_ids: + for youtube_id in self.to_delete: + YoutubeVideo(youtube_id).delete_media_file() + + def index(self): + """index new""" + if not self.to_index: + print("nothing to index") return - total = len(video_ids) - for idx, youtube_id in enumerate(video_ids): + total = len(self.to_index) + for idx, youtube_id in enumerate(self.to_index): if self.task: self.task.send_progress( message_lines=[ @@ -202,4 +89,36 @@ class Filesystem(ScannerBase): ) index_new_video(youtube_id) - CommentList(video_ids, task=self.task).index() + CommentList(self.to_index, task=self.task).index() + + def url_fix(self): + """ + update path v0.3.6 to v0.3.7 + fix url not matching channel-videoid pattern + """ + bool_must = ( + "doc['media_url'].value == " + + "(doc['channel.channel_id'].value + '/' + " + + "doc['youtube_id'].value) + '.mp4'" + ) + to_update = ( + "ctx._source['media_url'] = " + + "ctx._source.channel['channel_id'] + '/' + " + + "ctx._source['youtube_id'] + '.mp4'" + ) + data = { + "query": { + "bool": { + "must_not": [{"script": {"script": {"source": bool_must}}}] + } + }, + "script": {"source": to_update}, + } + response, _ = ElasticWrap("ta_video/_update_by_query").post(data=data) + updated = response.get("updates") + if updated: + print(f"updated {updated} bad media_url") + if self.task: + self.task.send_progress( + [f"Updated {updated} wrong media urls."] + ) diff --git a/tubearchivist/home/tasks.py b/tubearchivist/home/tasks.py index 1251891..c590783 100644 --- a/tubearchivist/home/tasks.py +++ b/tubearchivist/home/tasks.py @@ -19,7 +19,7 @@ from home.src.download.yt_dlp_handler import VideoDownloader from home.src.es.backup import ElasticBackup from home.src.es.index_setup import ElasitIndexWrap from home.src.index.channel import YoutubeChannel -from home.src.index.filesystem import Filesystem +from home.src.index.filesystem import Scanner from home.src.index.manual import ImportFolderScanner from home.src.index.reindex import Reindex, ReindexManual, ReindexPopulate from home.src.ta.config import AppConfig, ReleaseVersion, ScheduleBuilder @@ -290,7 +290,9 @@ def rescan_filesystem(self): return manager.init(self) - Filesystem(task=self).process() + handler = Scanner(task=self) + handler.scan() + handler.apply() ThumbValidator(task=self).validate() From 66a14cf389334c11a7d424dc4c85d2ba90b94be0 Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 20 May 2023 16:08:43 +0700 Subject: [PATCH 09/38] remove unused clean_string --- tubearchivist/home/src/ta/helper.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/tubearchivist/home/src/ta/helper.py b/tubearchivist/home/src/ta/helper.py index 9934598..6016d3c 100644 --- a/tubearchivist/home/src/ta/helper.py +++ b/tubearchivist/home/src/ta/helper.py @@ -6,25 +6,13 @@ Loose collection of helper functions import json import os import random -import re import string -import unicodedata from datetime import datetime from urllib.parse import urlparse import requests -def clean_string(file_name: str) -> str: - """clean string to only asci characters""" - whitelist = "-_.() " + string.ascii_letters + string.digits - normalized = unicodedata.normalize("NFKD", file_name) - ascii_only = normalized.encode("ASCII", "ignore").decode().strip() - white_listed: str = "".join(c for c in ascii_only if c in whitelist) - cleaned: str = re.sub(r"[ ]{2,}", " ", white_listed) - return cleaned - - def ignore_filelist(filelist: list[str]) -> list[str]: """ignore temp files for os.listdir sanitizer""" to_ignore = ["Icon\r\r", "Temporary Items", "Network Trash Folder"] From 139d20560f40c755ea6a4c4a38dc1522b35c73cc Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 20 May 2023 16:30:19 +0700 Subject: [PATCH 10/38] remove unused channel folder refresh --- tubearchivist/home/src/index/reindex.py | 69 ------------------------- 1 file changed, 69 deletions(-) diff --git a/tubearchivist/home/src/index/reindex.py b/tubearchivist/home/src/index/reindex.py index b5c8459..8e86577 100644 --- a/tubearchivist/home/src/index/reindex.py +++ b/tubearchivist/home/src/index/reindex.py @@ -6,7 +6,6 @@ functionality: import json import os -import shutil from datetime import datetime from time import sleep @@ -14,7 +13,6 @@ from home.src.download.queue import PendingList from home.src.download.subscriptions import ChannelSubscription from home.src.download.thumbnails import ThumbManager from home.src.download.yt_dlp_base import CookieHandler -from home.src.download.yt_dlp_handler import VideoDownloader from home.src.es.connect import ElasticWrap, IndexPaginate from home.src.index.channel import YoutubeChannel from home.src.index.comments import Comments @@ -276,14 +274,6 @@ class Reindex(ReindexBase): self.task.send_progress(message, progress=progress) def _reindex_single_video(self, youtube_id): - """wrapper to handle channel name changes""" - try: - self._reindex_single_video_call(youtube_id) - except FileNotFoundError: - ChannelUrlFixer(youtube_id, self.config).run() - self._reindex_single_video_call(youtube_id) - - def _reindex_single_video_call(self, youtube_id): """refresh data for single video""" video = YoutubeVideo(youtube_id) @@ -467,65 +457,6 @@ class ReindexProgress(ReindexBase): return state_dict -class ChannelUrlFixer: - """fix not matching channel names in reindex""" - - def __init__(self, youtube_id, config): - self.youtube_id = youtube_id - self.config = config - self.video = False - - def run(self): - """check and run if needed""" - print(f"{self.youtube_id}: failed to build channel path, try to fix.") - video_path_is, video_folder_is = self.get_as_is() - if not os.path.exists(video_path_is): - print(f"giving up reindex, video in video: {self.video.json_data}") - raise ValueError - - _, video_folder_should = self.get_as_should() - - if video_folder_is != video_folder_should: - self.process(video_path_is) - else: - print(f"{self.youtube_id}: skip channel url fixer") - - def get_as_is(self): - """get video object as is""" - self.video = YoutubeVideo(self.youtube_id) - self.video.get_from_es() - video_path_is = os.path.join( - self.config["application"]["videos"], - self.video.json_data["media_url"], - ) - video_folder_is = os.path.split(video_path_is)[0] - - return video_path_is, video_folder_is - - def get_as_should(self): - """add fresh metadata from remote""" - self.video.get_from_youtube() - self.video.add_file_path() - - video_path_should = os.path.join( - self.config["application"]["videos"], - self.video.json_data["media_url"], - ) - video_folder_should = os.path.split(video_path_should)[0] - return video_path_should, video_folder_should - - def process(self, video_path_is): - """fix filepath""" - print(f"{self.youtube_id}: fixing channel rename.") - cache_dir = self.config["application"]["cache_dir"] - new_path = os.path.join( - cache_dir, "download", self.youtube_id + ".mp4" - ) - shutil.move(video_path_is, new_path, copy_function=shutil.copyfile) - VideoDownloader().move_to_archive(self.video.json_data) - self.video.update_media_url() - - class ChannelFullScan: """ update from v0.3.0 to v0.3.1 From c4e2332b831e6545308ceefc66fdb073f42dad4b Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 20 May 2023 19:29:32 +0700 Subject: [PATCH 11/38] fix startup race condition cluster health --- tubearchivist/config/management/commands/ta_connection.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tubearchivist/config/management/commands/ta_connection.py b/tubearchivist/config/management/commands/ta_connection.py index a6159b3..86a3bd3 100644 --- a/tubearchivist/config/management/commands/ta_connection.py +++ b/tubearchivist/config/management/commands/ta_connection.py @@ -86,6 +86,8 @@ class Command(BaseCommand): continue if status_code and status_code == 200: + path = "_cluster/health?wait_for_status=yellow&timeout=30s" + _, _ = ElasticWrap(path).get() self.stdout.write( self.style.SUCCESS(" ✓ ES connection established") ) @@ -116,7 +118,7 @@ class Command(BaseCommand): return message = ( - " 🗙 ES connection failed. " + " 🗙 ES version check failed. " + f"Expected {self.MIN_MAJOR}.{self.MIN_MINOR} but got {version}" ) self.stdout.write(self.style.ERROR(f"{message}")) From 868247e6d478c1225e4a010ba5377c950d7fb33a Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 20 May 2023 19:30:40 +0700 Subject: [PATCH 12/38] add startup folder migration command --- docker_assets/run.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/docker_assets/run.sh b/docker_assets/run.sh index 686624e..618b0f3 100644 --- a/docker_assets/run.sh +++ b/docker_assets/run.sh @@ -14,6 +14,7 @@ fi python manage.py ta_envcheck python manage.py ta_connection python manage.py ta_startup +python manage.py ta_migpath # start all tasks nginx & From 904d0de6aab9c46f8e79b5f3f338e3fd6f6368a7 Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 20 May 2023 19:37:41 +0700 Subject: [PATCH 13/38] fix linter --- tubearchivist/home/src/index/filesystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubearchivist/home/src/index/filesystem.py b/tubearchivist/home/src/index/filesystem.py index 7594ee0..6935ff4 100644 --- a/tubearchivist/home/src/index/filesystem.py +++ b/tubearchivist/home/src/index/filesystem.py @@ -50,7 +50,7 @@ class Scanner: data = {"query": {"match_all": {}}, "_source": ["youtube_id"]} response = IndexPaginate("ta_video", data).get_results() - return set(i["youtube_id"] for i in response) + return {i["youtube_id"] for i in response} def apply(self): """apply all changes""" From 7e2cd6acd3f1727413e286980370b2b3eab30366 Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 20 May 2023 19:41:33 +0700 Subject: [PATCH 14/38] fix linter, take 2 --- tubearchivist/home/src/index/filesystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubearchivist/home/src/index/filesystem.py b/tubearchivist/home/src/index/filesystem.py index 6935ff4..e023444 100644 --- a/tubearchivist/home/src/index/filesystem.py +++ b/tubearchivist/home/src/index/filesystem.py @@ -39,7 +39,7 @@ class Scanner: for channel in channels: folder = os.path.join(self.VIDEOS, channel) files = ignore_filelist(os.listdir(folder)) - downloaded.update(set(i.split(".")[0] for i in files)) + downloaded.update({i.split(".")[0] for i in files}) return downloaded From 7082718c148c814f75bc6fff31ceba58fd39d539 Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 20 May 2023 20:08:36 +0700 Subject: [PATCH 15/38] add days to seconds string converter --- tubearchivist/home/src/index/video_streams.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tubearchivist/home/src/index/video_streams.py b/tubearchivist/home/src/index/video_streams.py index a439716..7f6f2f2 100644 --- a/tubearchivist/home/src/index/video_streams.py +++ b/tubearchivist/home/src/index/video_streams.py @@ -35,24 +35,27 @@ class DurationConverter: return duration_sec @staticmethod - def get_str(duration_sec): + def get_str(seconds): """takes duration in sec and returns clean string""" - if not duration_sec: + if not seconds: # failed to extract return "NA" - hours = int(duration_sec // 3600) - minutes = int((duration_sec - (hours * 3600)) // 60) - secs = int(duration_sec - (hours * 3600) - (minutes * 60)) + days = int(seconds // (24 * 3600)) + hours = int((seconds % (24 * 3600)) // 3600) + minutes = int((seconds % 3600) // 60) + seconds = int(seconds % 60) duration_str = str() + if days: + duration_str = f"{days}d " if hours: - duration_str = str(hours).zfill(2) + ":" + duration_str = duration_str + str(hours).zfill(2) + ":" if minutes: duration_str = duration_str + str(minutes).zfill(2) + ":" else: duration_str = duration_str + "00:" - duration_str = duration_str + str(secs).zfill(2) + duration_str = duration_str + str(seconds).zfill(2) return duration_str From 5e92d06f21dff75de9cc6d81574c2681ef25a934 Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 20 May 2023 21:25:50 +0700 Subject: [PATCH 16/38] fix dl error retry logic, store and return error, #477 --- tubearchivist/home/src/download/yt_dlp_base.py | 6 +++--- tubearchivist/home/src/download/yt_dlp_handler.py | 13 +++++++++++-- tubearchivist/home/src/es/index_mapping.json | 3 +++ tubearchivist/home/templates/home/downloads.html | 3 +++ 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/tubearchivist/home/src/download/yt_dlp_base.py b/tubearchivist/home/src/download/yt_dlp_base.py index 5cecc35..62e5b02 100644 --- a/tubearchivist/home/src/download/yt_dlp_base.py +++ b/tubearchivist/home/src/download/yt_dlp_base.py @@ -48,11 +48,11 @@ class YtWrap: with yt_dlp.YoutubeDL(self.obs) as ydl: try: ydl.download([url]) - except yt_dlp.utils.DownloadError: + except yt_dlp.utils.DownloadError as err: print(f"{url}: failed to download.") - return False + return False, str(err) - return True + return True, True def extract(self, url): """make extract request""" diff --git a/tubearchivist/home/src/download/yt_dlp_handler.py b/tubearchivist/home/src/download/yt_dlp_handler.py index 0ea813a..abd3a94 100644 --- a/tubearchivist/home/src/download/yt_dlp_handler.py +++ b/tubearchivist/home/src/download/yt_dlp_handler.py @@ -203,12 +203,13 @@ class VideoDownloader: def _get_next(self, auto_only): """get next item in queue""" must_list = [{"term": {"status": {"value": "pending"}}}] + must_not_list = [{"exists": {"field": "message"}}] if auto_only: must_list.append({"term": {"auto_start": {"value": True}}}) data = { "size": 1, - "query": {"bool": {"must": must_list}}, + "query": {"bool": {"must": must_list, "must_not": must_not_list}}, "sort": [ {"auto_start": {"order": "desc"}}, {"timestamp": {"order": "asc"}}, @@ -344,7 +345,9 @@ class VideoDownloader: if youtube_id in file_name: obs["outtmpl"] = os.path.join(dl_cache, file_name) - success = YtWrap(obs, self.config).download(youtube_id) + success, message = YtWrap(obs, self.config).download(youtube_id) + if not success: + self._handle_error(youtube_id, message) if self.obs["writethumbnail"]: # webp files don't get cleaned up automatically @@ -356,6 +359,12 @@ class VideoDownloader: return success + @staticmethod + def _handle_error(youtube_id, message): + """store error message""" + data = {"doc": {"message": message}} + _, _ = ElasticWrap(f"ta_download/_update/{youtube_id}").post(data=data) + def move_to_archive(self, vid_dict): """move downloaded video from cache to archive""" videos = self.config["application"]["videos"] diff --git a/tubearchivist/home/src/es/index_mapping.json b/tubearchivist/home/src/es/index_mapping.json index 34baf87..3bf7f38 100644 --- a/tubearchivist/home/src/es/index_mapping.json +++ b/tubearchivist/home/src/es/index_mapping.json @@ -380,6 +380,9 @@ }, "auto_start": { "type": "boolean" + }, + "message": { + "type": "text" } }, "expected_set": { diff --git a/tubearchivist/home/templates/home/downloads.html b/tubearchivist/home/templates/home/downloads.html index 93eace6..1b156fb 100644 --- a/tubearchivist/home/templates/home/downloads.html +++ b/tubearchivist/home/templates/home/downloads.html @@ -97,6 +97,9 @@

{{ video.source.title }}

Published: {{ video.source.published }} | Duration: {{ video.source.duration }} | {{ video.source.youtube_id }}

+ {% if video.source.message %} +

{{ video.source.message }}

+ {% endif %}
{% if show_ignored_only %} From 6fb788b259e3661b14ad75bba9c3739544b8f085 Mon Sep 17 00:00:00 2001 From: simon Date: Mon, 22 May 2023 17:34:49 +0700 Subject: [PATCH 17/38] add delete button for unavailable video --- tubearchivist/home/templates/home/downloads.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tubearchivist/home/templates/home/downloads.html b/tubearchivist/home/templates/home/downloads.html index 1b156fb..2084358 100644 --- a/tubearchivist/home/templates/home/downloads.html +++ b/tubearchivist/home/templates/home/downloads.html @@ -108,6 +108,9 @@ {% endif %} + {% if video.source.message %} + + {% endif %}
From 5927ced4850c5dde4d47ac018925c511156d345c Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 27 May 2023 18:26:34 +0700 Subject: [PATCH 18/38] bump libs --- tubearchivist/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tubearchivist/requirements.txt b/tubearchivist/requirements.txt index 8b26b8f..57f5e44 100644 --- a/tubearchivist/requirements.txt +++ b/tubearchivist/requirements.txt @@ -1,11 +1,11 @@ celery==5.2.7 Django==4.2.1 django-auth-ldap==4.3.0 -django-cors-headers==3.14.0 +django-cors-headers==4.0.0 djangorestframework==3.14.0 Pillow==9.5.0 -redis==4.5.4 -requests==2.30.0 +redis==4.5.5 +requests==2.31.0 ryd-client==0.0.6 uWSGI==2.0.21 whitenoise==6.4.0 From 247808563aa236bd0c3749d04da1f370f7780419 Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 16 Jun 2023 15:47:38 +0700 Subject: [PATCH 19/38] download error recovering --- tubearchivist/home/src/download/queue.py | 8 +++++++- tubearchivist/home/src/download/yt_dlp_base.py | 18 +++++++++++++++--- tubearchivist/home/tasks.py | 2 +- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/tubearchivist/home/src/download/queue.py b/tubearchivist/home/src/download/queue.py index 006b7a0..97636bb 100644 --- a/tubearchivist/home/src/download/queue.py +++ b/tubearchivist/home/src/download/queue.py @@ -114,7 +114,13 @@ class PendingInteract: def update_status(self): """update status of pending item""" if self.status == "priority": - data = {"doc": {"status": "pending", "auto_start": True}} + data = { + "doc": { + "status": "pending", + "auto_start": True, + "message": None, + } + } else: data = {"doc": {"status": self.status}} diff --git a/tubearchivist/home/src/download/yt_dlp_base.py b/tubearchivist/home/src/download/yt_dlp_base.py index 62e5b02..526dbd4 100644 --- a/tubearchivist/home/src/download/yt_dlp_base.py +++ b/tubearchivist/home/src/download/yt_dlp_base.py @@ -49,7 +49,10 @@ class YtWrap: try: ydl.download([url]) except yt_dlp.utils.DownloadError as err: - print(f"{url}: failed to download.") + print(f"{url}: failed to download with message {err}") + if "Temporary failure in name resolution" in str(err): + raise ConnectionError("lost the internet, abort!") from err + return False, str(err) return True, True @@ -61,8 +64,17 @@ class YtWrap: except cookiejar.LoadError: print("cookie file is invalid") return False - except (yt_dlp.utils.ExtractorError, yt_dlp.utils.DownloadError): - print(f"{url}: failed to get info from youtube") + except yt_dlp.utils.ExtractorError as err: + print(f"{url}: failed to extract with message: {err}, continue...") + return False + except yt_dlp.utils.DownloadError as err: + if "This channel does not have a" in str(err): + return False + + print(f"{url}: failed to get info from youtube with message {err}") + if "Temporary failure in name resolution" in str(err): + raise ConnectionError("lost the internet, abort!") from err + return False return response diff --git a/tubearchivist/home/tasks.py b/tubearchivist/home/tasks.py index c590783..76bc888 100644 --- a/tubearchivist/home/tasks.py +++ b/tubearchivist/home/tasks.py @@ -113,7 +113,7 @@ class BaseTask(Task): """callback for task failure""" print(f"{task_id} Failed callback") message, key = self._build_message(level="error") - message.update({"messages": ["Task failed"]}) + message.update({"messages": [f"Task failed: {exc}"]}) RedisArchivist().set_message(key, message, expire=20) def on_success(self, retval, task_id, args, kwargs): From 094ccf4186aad05c11eae130dde7e04146126e98 Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 16 Jun 2023 15:48:02 +0700 Subject: [PATCH 20/38] bump libs --- tubearchivist/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tubearchivist/requirements.txt b/tubearchivist/requirements.txt index 57f5e44..d3c414e 100644 --- a/tubearchivist/requirements.txt +++ b/tubearchivist/requirements.txt @@ -1,7 +1,7 @@ -celery==5.2.7 -Django==4.2.1 +celery==5.3.0 +Django==4.2.2 django-auth-ldap==4.3.0 -django-cors-headers==4.0.0 +django-cors-headers==4.1.0 djangorestframework==3.14.0 Pillow==9.5.0 redis==4.5.5 From 103409770d20e472bccf3ab50e3a1cbad4e7669b Mon Sep 17 00:00:00 2001 From: Simon Date: Thu, 22 Jun 2023 23:27:16 +0700 Subject: [PATCH 21/38] temporary fix for is_favorited extraction error --- tubearchivist/home/src/index/comments.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tubearchivist/home/src/index/comments.py b/tubearchivist/home/src/index/comments.py index 1cbe75a..b67735d 100644 --- a/tubearchivist/home/src/index/comments.py +++ b/tubearchivist/home/src/index/comments.py @@ -120,7 +120,9 @@ class Comments: "comment_timestamp": comment["timestamp"], "comment_time_text": time_text, "comment_likecount": comment["like_count"], - "comment_is_favorited": comment["is_favorited"], + "comment_is_favorited": comment.get( + "is_favorited" + ), # temporary fix for yt-dlp upstream issue 7389 "comment_author": comment["author"], "comment_author_id": comment["author_id"], "comment_author_thumbnail": comment["author_thumbnail"], From 32721cf7ed35ca7320d240367b3c7f0e376e9ebd Mon Sep 17 00:00:00 2001 From: Simon Date: Thu, 22 Jun 2023 23:27:48 +0700 Subject: [PATCH 22/38] bump base python version --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index db55bf2..195c66c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ # First stage to build python wheel -FROM python:3.10.9-slim-bullseye AS builder +FROM python:3.11.3-slim-bullseye AS builder ARG TARGETPLATFORM RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -14,7 +14,7 @@ COPY ./tubearchivist/requirements.txt /requirements.txt RUN pip install --user -r requirements.txt # build final image -FROM python:3.10.9-slim-bullseye as tubearchivist +FROM python:3.11.3-slim-bullseye as tubearchivist ARG TARGETPLATFORM ARG INSTALL_DEBUG From 3f1d8cf75d6a0e96e65141bcabf5fc1ea541e17c Mon Sep 17 00:00:00 2001 From: Simon Date: Thu, 22 Jun 2023 23:28:06 +0700 Subject: [PATCH 23/38] add .venv --- .gitignore | 1 + deploy.sh | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 14fd5ce..682dd83 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # python testing cache __pycache__ +.venv # django testing db db.sqlite3 diff --git a/deploy.sh b/deploy.sh index bbc27d5..191cbfb 100755 --- a/deploy.sh +++ b/deploy.sh @@ -49,6 +49,7 @@ function sync_test { --exclude ".gitignore" \ --exclude "**/cache" \ --exclude "**/__pycache__/" \ + --exclude ".venv" \ --exclude "db.sqlite3" \ --exclude ".mypy_cache" \ . -e ssh "$host":tubearchivist @@ -87,14 +88,14 @@ function validate { # note: this logic is duplicated in the `./github/workflows/lint_python.yml` config # if you update this file, you should update that as well echo "running black" - black --exclude "migrations/*" --diff --color --check -l 79 "$check_path" + black --force-exclude "migrations/*" --diff --color --check -l 79 "$check_path" echo "running codespell" - codespell --skip="./.git,./package.json,./package-lock.json,./node_modules,./.mypy_cache" "$check_path" + codespell --skip="./.git,./.venv,./package.json,./package-lock.json,./node_modules,./.mypy_cache" "$check_path" echo "running flake8" - flake8 "$check_path" --exclude "migrations" --count --max-complexity=10 \ + flake8 "$check_path" --exclude "migrations,.venv" --count --max-complexity=10 \ --max-line-length=79 --show-source --statistics echo "running isort" - isort --skip "migrations" --check-only --diff --profile black -l 79 "$check_path" + isort --skip "migrations" --skip ".venv" --check-only --diff --profile black -l 79 "$check_path" printf " \n> all validations passed\n" } From 879497d25aff0b8f4c53769e5c2a2928ddf25968 Mon Sep 17 00:00:00 2001 From: Simon Date: Thu, 22 Jun 2023 23:28:17 +0700 Subject: [PATCH 24/38] bump libs --- tubearchivist/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tubearchivist/requirements.txt b/tubearchivist/requirements.txt index d3c414e..58d1950 100644 --- a/tubearchivist/requirements.txt +++ b/tubearchivist/requirements.txt @@ -1,4 +1,4 @@ -celery==5.3.0 +celery==5.3.1 Django==4.2.2 django-auth-ldap==4.3.0 django-cors-headers==4.1.0 @@ -8,5 +8,5 @@ redis==4.5.5 requests==2.31.0 ryd-client==0.0.6 uWSGI==2.0.21 -whitenoise==6.4.0 -yt_dlp==2023.3.4 +whitenoise==6.5.0 +yt_dlp==2023.6.22 From 88e199ef9cc19b362c10de1278783d9eb1657202 Mon Sep 17 00:00:00 2001 From: Simon Date: Thu, 22 Jun 2023 23:29:05 +0700 Subject: [PATCH 25/38] reset reindex counter on new added to queue, #478 --- tubearchivist/home/src/index/reindex.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/tubearchivist/home/src/index/reindex.py b/tubearchivist/home/src/index/reindex.py index 8e86577..99fd0f8 100644 --- a/tubearchivist/home/src/index/reindex.py +++ b/tubearchivist/home/src/index/reindex.py @@ -52,6 +52,7 @@ class ReindexBase: def __init__(self): self.config = AppConfig().config self.now = int(datetime.now().timestamp()) + self.total = None def populate(self, all_ids, reindex_config): """add all to reindex ids to redis queue""" @@ -59,6 +60,7 @@ class ReindexBase: return RedisQueue(queue_name=reindex_config["queue_name"]).add_list(all_ids) + self.total = None class ReindexPopulate(ReindexBase): @@ -236,19 +238,19 @@ class Reindex(ReindexBase): if not RedisQueue(index_config["queue_name"]).has_item(): continue - total = RedisQueue(index_config["queue_name"]).length() + self.total = RedisQueue(index_config["queue_name"]).length() while True: - has_next = self.reindex_index(name, index_config, total) + has_next = self.reindex_index(name, index_config) if not has_next: break - def reindex_index(self, name, index_config, total): + def reindex_index(self, name, index_config): """reindex all of a single index""" reindex = self.get_reindex_map(index_config["index_name"]) youtube_id = RedisQueue(index_config["queue_name"]).get_next() if youtube_id: if self.task: - self._notify(name, index_config, total) + self._notify(name, index_config) reindex(youtube_id) sleep_interval = self.config["downloads"].get("sleep_interval", 0) sleep(sleep_interval) @@ -265,12 +267,15 @@ class Reindex(ReindexBase): return def_map.get(index_name) - def _notify(self, name, index_config, total): + def _notify(self, name, index_config): """send notification back to task""" + if self.total is None: + self.total = RedisQueue(index_config["queue_name"]).length() + remaining = RedisQueue(index_config["queue_name"]).length() - idx = total - remaining - message = [f"Reindexing {name.title()}s {idx}/{total}"] - progress = idx / total + idx = self.total - remaining + message = [f"Reindexing {name.title()}s {idx}/{self.total}"] + progress = idx / self.total self.task.send_progress(message, progress=progress) def _reindex_single_video(self, youtube_id): From 72a98b0473639d9e973105d70e574e82a635522e Mon Sep 17 00:00:00 2001 From: Simon Date: Thu, 22 Jun 2023 23:36:54 +0700 Subject: [PATCH 26/38] handle missing channel_tvart_url in thumb validator, #479 --- tubearchivist/home/src/download/thumbnails.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubearchivist/home/src/download/thumbnails.py b/tubearchivist/home/src/download/thumbnails.py index 256a746..88b6558 100644 --- a/tubearchivist/home/src/download/thumbnails.py +++ b/tubearchivist/home/src/download/thumbnails.py @@ -270,7 +270,7 @@ class ValidatorCallback: urls = ( channel["_source"]["channel_thumb_url"], channel["_source"]["channel_banner_url"], - channel["_source"]["channel_tvart_url"], + channel["_source"].get("channel_tvart_url", False), ) handler = ThumbManager(channel["_source"]["channel_id"]) handler.download_channel_art(urls, skip_existing=True) From 061c653bce6fff3ee2ed2ecb611eb9b232d2353a Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 23 Jun 2023 00:15:07 +0700 Subject: [PATCH 27/38] retry get config better startup error handling, #485 --- tubearchivist/home/src/ta/config.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tubearchivist/home/src/ta/config.py b/tubearchivist/home/src/ta/config.py index f191eae..f9b3474 100644 --- a/tubearchivist/home/src/ta/config.py +++ b/tubearchivist/home/src/ta/config.py @@ -8,6 +8,7 @@ import json import os import re from random import randint +from time import sleep import requests from celery.schedules import crontab @@ -67,11 +68,19 @@ class AppConfig: @staticmethod def get_config_redis(): """read config json set from redis to overwrite defaults""" - config = RedisArchivist().get_message("config") - if not list(config.values())[0]: - return False + for i in range(10): + try: + config = RedisArchivist().get_message("config") + if not list(config.values())[0]: + return False - return config + return config + + except Exception: # pylint: disable=broad-except + print(f"... Redis connection failed, retry [{i}/10]") + sleep(3) + + raise ConnectionError("failed to connect to redis") def update_config(self, form_post): """update config values from settings form""" From 1be80b24c2cf4426658a43386b914bbe7eaf672d Mon Sep 17 00:00:00 2001 From: lamusmaser <1940060+lamusmaser@users.noreply.github.com> Date: Tue, 27 Jun 2023 21:50:28 -0600 Subject: [PATCH 28/38] Implement #490 - Add version API and add local_version function. (#491) * Add version API and add local_version function. * Minor adjustments for linting. * Add missing newlines for linter. * Add missing comma to `urls.py`. * Remove `version/` endpoint. * Remove the `VersionView`. * Prepare `PingView` for removal of the `is_static` response. * Remove the `is_unstable` response from `ReleaseVersion`. * Readd missing class instantiation for first call of `ReleaseVersion`. --- tubearchivist/api/views.py | 8 ++++++-- tubearchivist/home/src/ta/config.py | 4 ++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index 15fde1d..357bd50 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -11,7 +11,7 @@ from home.src.index.channel import YoutubeChannel from home.src.index.generic import Pagination from home.src.index.reindex import ReindexProgress from home.src.index.video import SponsorBlock, YoutubeVideo -from home.src.ta.config import AppConfig +from home.src.ta.config import AppConfig, ReleaseVersion from home.src.ta.ta_redis import RedisArchivist from home.src.ta.task_manager import TaskCommand, TaskManager from home.src.ta.urlparser import Parser @@ -535,7 +535,11 @@ class PingView(ApiBaseView): @staticmethod def get(request): """get pong""" - data = {"response": "pong", "user": request.user.id} + data = { + "response": "pong", + "user": request.user.id, + "version": ReleaseVersion().get_local_version(), + } return Response(data) diff --git a/tubearchivist/home/src/ta/config.py b/tubearchivist/home/src/ta/config.py index f9b3474..671602c 100644 --- a/tubearchivist/home/src/ta/config.py +++ b/tubearchivist/home/src/ta/config.py @@ -326,6 +326,10 @@ class ReleaseVersion: RedisArchivist().set_message(self.NEW_KEY, message) print(f"[{self.local_version}]: found new version {new_version}") + def get_local_version(self): + """read version from local""" + return self.local_version + def get_remote_version(self): """read version from remote""" self.response = requests.get(self.REMOTE_URL, timeout=20).json() From 8a7cb8bc6f974d5df7490814ce77be54225c8f98 Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 28 Jun 2023 20:07:17 +0700 Subject: [PATCH 29/38] bump redis --- tubearchivist/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubearchivist/requirements.txt b/tubearchivist/requirements.txt index 58d1950..045aefd 100644 --- a/tubearchivist/requirements.txt +++ b/tubearchivist/requirements.txt @@ -4,7 +4,7 @@ django-auth-ldap==4.3.0 django-cors-headers==4.1.0 djangorestframework==3.14.0 Pillow==9.5.0 -redis==4.5.5 +redis==4.6.0 requests==2.31.0 ryd-client==0.0.6 uWSGI==2.0.21 From 2a60360f4a02ba760665dd92f425d467005df0df Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 28 Jun 2023 20:07:40 +0700 Subject: [PATCH 30/38] handle empty channel migration cleanup --- tubearchivist/config/management/commands/ta_migpath.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tubearchivist/config/management/commands/ta_migpath.py b/tubearchivist/config/management/commands/ta_migpath.py index 4d039a5..78b4c2f 100644 --- a/tubearchivist/config/management/commands/ta_migpath.py +++ b/tubearchivist/config/management/commands/ta_migpath.py @@ -178,6 +178,9 @@ class ChannelMigration: def delete_old(self, all_videos): """delete old folder path if empty""" + if not all_videos: + return + channel_name = os.path.split(all_videos[0]["media_url"])[0] old_path = os.path.join(self.videos, channel_name) if os.path.exists(old_path) and not os.listdir(old_path): From 61b04ba5cf7ffb6fd20a4b91332c4186d659f70b Mon Sep 17 00:00:00 2001 From: Simon Date: Mon, 24 Jul 2023 10:51:13 +0700 Subject: [PATCH 31/38] channel migration take 2 --- .../config/management/commands/ta_migpath.py | 153 ++++++++---------- 1 file changed, 67 insertions(+), 86 deletions(-) diff --git a/tubearchivist/config/management/commands/ta_migpath.py b/tubearchivist/config/management/commands/ta_migpath.py index 78b4c2f..f407800 100644 --- a/tubearchivist/config/management/commands/ta_migpath.py +++ b/tubearchivist/config/management/commands/ta_migpath.py @@ -25,99 +25,83 @@ class Command(BaseCommand): def handle(self, *args, **options): """run commands""" self.stdout.write(TOPIC) - need_migration = self.channels_need_migration() - if not need_migration: + + handler = FolderMigration() + to_migrate = handler.get_to_migrate() + if not to_migrate: self.stdout.write( - self.style.SUCCESS(" no channel migration needed") + self.style.SUCCESS(" no channel migration needed\n") ) return + self.stdout.write(self.style.SUCCESS(" migrating channels")) + total_channels = handler.create_folders(to_migrate) self.stdout.write( - self.style.SUCCESS(f" migrating {len(need_migration)} channels") - ) - for channel in need_migration: - channel_name = channel["channel_name"] - channel_id = channel["channel_id"] - self.stdout.write( - self.style.SUCCESS( - f" migrating {channel_name} [{channel_id}]" - ) - ) - ChannelMigration(channel).migrate() - - self.stdout.write( - self.style.SUCCESS(" ✓ channel migration completed") + self.style.SUCCESS(f" created {total_channels} channels") ) - def channels_need_migration(self): - """get channels that need migration""" - all_indexed = self._get_channel_indexed() - all_folders = self._get_channel_folders() - need_migration = [] - for channel in all_indexed: - if channel["channel_id"] not in all_folders: - need_migration.append(channel) + self.stdout.write( + self.style.SUCCESS(f" migrating {len(to_migrate)} videos") + ) + handler.migrate_videos(to_migrate) + self.stdout.write(self.style.SUCCESS(" update videos in index")) + handler.send_bulk() - return need_migration + self.stdout.write(self.style.SUCCESS(" cleanup old folders")) + handler.delete_old() - def _get_channel_indexed(self): - """get all channels indexed""" - all_results = IndexPaginate("ta_channel", False).get_results() - - return all_results - - def _get_channel_folders(self): - """get all channel folders""" - base_folder = AppConfig().config["application"]["videos"] - existing_folders = ignore_filelist(os.listdir(base_folder)) - - return existing_folders + self.stdout.write(self.style.SUCCESS(" ✓ migration completed\n")) -class ChannelMigration: - """migrate single channel""" +class FolderMigration: + """migrate video archive folder""" - def __init__(self, channel): - self.channel = channel + def __init__(self): self.config = AppConfig().config self.videos = self.config["application"]["videos"] self.bulk_list = [] - def migrate(self): - """run migration""" - self._create_new_folder() - all_videos = self.get_channel_videos() - self.migrate_videos(all_videos) - self.send_bulk() - self.delete_old(all_videos) + def get_to_migrate(self): + """get videos to migrate""" + script = ( + "doc['media_url'].value == " + + "doc['channel.channel_id'].value + '/'" + + " + doc['youtube_id'].value + '.mp4'" + ) + data = { + "query": {"bool": {"must_not": [{"script": {"script": script}}]}}, + "_source": [ + "youtube_id", + "media_url", + "channel.channel_id", + "subtitles", + ], + } + response = IndexPaginate("ta_video", data).get_results() - def _create_new_folder(self): - """create new channel id folder""" + return response + + def create_folders(self, to_migrate): + """create required channel folders""" host_uid = self.config["application"]["HOST_UID"] host_gid = self.config["application"]["HOST_GID"] - new_path = os.path.join(self.videos, self.channel["channel_id"]) - if not os.path.exists(new_path): - os.mkdir(new_path) + all_channel_ids = {i["channel"]["channel_id"] for i in to_migrate} + + for channel_id in all_channel_ids: + new_folder = os.path.join(self.videos, channel_id) + os.makedirs(new_folder, exist_ok=True) if host_uid and host_gid: - os.chown(new_path, host_uid, host_gid) + os.chown(new_folder, host_uid, host_gid) - def get_channel_videos(self): - """get all videos of channel""" - data = { - "query": { - "term": { - "channel.channel_id": {"value": self.channel["channel_id"]} - } - } - } - all_videos = IndexPaginate("ta_video", data).get_results() + return len(all_channel_ids) - return all_videos - - def migrate_videos(self, all_videos): + def migrate_videos(self, to_migrate): """migrate all videos of channel""" - for video in all_videos: + for video in to_migrate: new_media_url = self._move_video_file(video) + if not new_media_url: + continue + all_subtitles = self._move_subtitles(video) action = { "update": {"_id": video["youtube_id"], "_index": "ta_video"} @@ -137,9 +121,10 @@ class ChannelMigration: return False new_media_url = os.path.join( - self.channel["channel_id"], video["youtube_id"] + ".mp4" + video["channel"]["channel_id"], video["youtube_id"] + ".mp4" ) - os.rename(old_path, os.path.join(self.videos, new_media_url)) + new_path = os.path.join(self.videos, new_media_url) + os.rename(old_path, new_path) return new_media_url @@ -155,11 +140,12 @@ class ChannelMigration: print(f"did not find expected subtitle at {old_path}") continue - ext = ".".join(old_path.split(".")[-2:]) new_media_url = os.path.join( - self.channel["channel_id"], video["youtube_id"] + f".{ext}" + video["channel"]["channel_id"], + f"{video.get('youtube_id')}.{subtitle.get('lang')}.vtt", ) - os.rename(old_path, os.path.join(self.videos, new_media_url)) + new_path = os.path.join(self.videos, new_media_url) + os.rename(old_path, new_path) subtitle["media_url"] = new_media_url return all_subtitles @@ -176,15 +162,10 @@ class ChannelMigration: if not status == 200: print(response) - def delete_old(self, all_videos): - """delete old folder path if empty""" - if not all_videos: - return - - channel_name = os.path.split(all_videos[0]["media_url"])[0] - old_path = os.path.join(self.videos, channel_name) - if os.path.exists(old_path) and not os.listdir(old_path): - os.rmdir(old_path) - return - - print(f"failed to clean up old folder {old_path}") + def delete_old(self): + """delete old empty folders""" + all_folders = ignore_filelist(os.listdir(self.videos)) + for folder in all_folders: + folder_path = os.path.join(self.videos, folder) + if not ignore_filelist(os.listdir(folder_path)): + os.rmdir(folder_path) From 99baf64b11368c61e4ca45929746a1508d9dbdc2 Mon Sep 17 00:00:00 2001 From: Simon Date: Mon, 24 Jul 2023 10:51:37 +0700 Subject: [PATCH 32/38] update requirements --- tubearchivist/requirements.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tubearchivist/requirements.txt b/tubearchivist/requirements.txt index 045aefd..f251cff 100644 --- a/tubearchivist/requirements.txt +++ b/tubearchivist/requirements.txt @@ -1,12 +1,12 @@ celery==5.3.1 -Django==4.2.2 -django-auth-ldap==4.3.0 -django-cors-headers==4.1.0 +Django==4.2.3 +django-auth-ldap==4.4.0 +django-cors-headers==4.2.0 djangorestframework==3.14.0 -Pillow==9.5.0 +Pillow==10.0.0 redis==4.6.0 requests==2.31.0 ryd-client==0.0.6 uWSGI==2.0.21 whitenoise==6.5.0 -yt_dlp==2023.6.22 +yt_dlp==2023.7.6 From 6022bdd3cdf6d9877d41ad90d6243e047a617b86 Mon Sep 17 00:00:00 2001 From: Simon Date: Mon, 24 Jul 2023 11:27:19 +0700 Subject: [PATCH 33/38] fix doc string --- tubearchivist/api/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index 357bd50..3afe692 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -189,7 +189,7 @@ class VideoCommentView(ApiBaseView): class VideoSimilarView(ApiBaseView): """resolves to /api/video//similar/ - GET: return max 3 videos similar to this + GET: return max 6 videos similar to this """ search_base = "ta_video/_search/" From efde4b1142f9a5796cb027e873b19b2554fdccb5 Mon Sep 17 00:00:00 2001 From: Simon Date: Mon, 24 Jul 2023 12:11:21 +0700 Subject: [PATCH 34/38] skip subtitle if not processed yet, #496 --- tubearchivist/home/src/index/subtitle.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tubearchivist/home/src/index/subtitle.py b/tubearchivist/home/src/index/subtitle.py index e289fbe..18af56a 100644 --- a/tubearchivist/home/src/index/subtitle.py +++ b/tubearchivist/home/src/index/subtitle.py @@ -62,7 +62,12 @@ class YoutubeSubtitle: if not all_formats: return False - subtitle = [i for i in all_formats if i["ext"] == "json3"][0] + subtitle_json3 = [i for i in all_formats if i["ext"] == "json3"] + if not subtitle_json3: + print(f"{self.video.youtube_id}-{lang}: json3 not processed") + return False + + subtitle = subtitle_json3[0] subtitle.update( {"lang": lang, "source": "auto", "media_url": media_url} ) From 434aa97a8651225d2af45b287ed0db3a2b27b207 Mon Sep 17 00:00:00 2001 From: Simon Date: Mon, 24 Jul 2023 23:44:27 +0700 Subject: [PATCH 35/38] static cache file path building, #498 --- tubearchivist/home/src/index/reindex.py | 5 ++- tubearchivist/home/src/index/video.py | 44 ++++++++----------------- 2 files changed, 18 insertions(+), 31 deletions(-) diff --git a/tubearchivist/home/src/index/reindex.py b/tubearchivist/home/src/index/reindex.py index 99fd0f8..7c69b49 100644 --- a/tubearchivist/home/src/index/reindex.py +++ b/tubearchivist/home/src/index/reindex.py @@ -287,7 +287,10 @@ class Reindex(ReindexBase): es_meta = video.json_data.copy() # get new - video.build_json() + media_url = os.path.join( + self.config["application"]["videos"], es_meta["media_url"] + ) + video.build_json(media_path=media_url) if not video.youtube_meta: video.deactivate() return diff --git a/tubearchivist/home/src/index/video.py b/tubearchivist/home/src/index/video.py index 7b2da2d..87360ae 100644 --- a/tubearchivist/home/src/index/video.py +++ b/tubearchivist/home/src/index/video.py @@ -231,18 +231,24 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): def build_dl_cache_path(self): """find video path in dl cache""" cache_dir = self.app_conf["cache_dir"] - cache_path = f"{cache_dir}/download/" - all_cached = os.listdir(cache_path) - for file_cached in all_cached: - if self.youtube_id in file_cached: - vid_path = os.path.join(cache_path, file_cached) - return vid_path + video_id = self.json_data["youtube_id"] + cache_path = f"{cache_dir}/download/{video_id}.mp4" + if os.path.exists(cache_path): + return cache_path + + channel_path = os.path.join( + self.app_conf["videos"], + self.json_data["channel"]["channel_id"], + f"{video_id}.mp4", + ) + if os.path.exists(channel_path): + return channel_path raise FileNotFoundError def add_player(self, media_path=False): """add player information for new videos""" - vid_path = self._get_vid_path(media_path) + vid_path = media_path or self.build_dl_cache_path() duration_handler = DurationConverter() duration = duration_handler.get_sec(vid_path) @@ -259,7 +265,7 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): def add_streams(self, media_path=False): """add stream metadata""" - vid_path = self._get_vid_path(media_path) + vid_path = media_path or self.build_dl_cache_path() media = MediaStreamExtractor(vid_path) self.json_data.update( { @@ -268,28 +274,6 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): } ) - def _get_vid_path(self, media_path=False): - """get path of media file""" - if media_path: - return media_path - - try: - # when indexing from download task - vid_path = self.build_dl_cache_path() - except FileNotFoundError as err: - # when reindexing needs to handle title rename - channel = os.path.split(self.json_data["media_url"])[0] - channel_dir = os.path.join(self.app_conf["videos"], channel) - all_files = os.listdir(channel_dir) - for file in all_files: - if self.youtube_id in file and file.endswith(".mp4"): - vid_path = os.path.join(channel_dir, file) - break - else: - raise FileNotFoundError("could not find video file") from err - - return vid_path - def add_file_path(self): """build media_url for where file will be located""" self.json_data["media_url"] = os.path.join( From ddfab4a3417603bf13f776ce248704471ef37e1e Mon Sep 17 00:00:00 2001 From: Simon Date: Tue, 25 Jul 2023 00:04:18 +0700 Subject: [PATCH 36/38] update packages --- package-lock.json | 1048 +++++---------------------------------------- 1 file changed, 115 insertions(+), 933 deletions(-) diff --git a/package-lock.json b/package-lock.json index c32eaea..3ab3aec 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "tubearchivist_project", - "lockfileVersion": 2, + "lockfileVersion": 3, "requires": true, "packages": { "": { @@ -10,16 +10,49 @@ "prettier": "^2.7.1" } }, + "node_modules/@aashutoshrathi/word-wrap": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@aashutoshrathi/word-wrap/-/word-wrap-1.2.6.tgz", + "integrity": "sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/@eslint-community/eslint-utils": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz", + "integrity": "sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==", + "dev": true, + "dependencies": { + "eslint-visitor-keys": "^3.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "peerDependencies": { + "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" + } + }, + "node_modules/@eslint-community/regexpp": { + "version": "4.6.1", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.6.1.tgz", + "integrity": "sha512-O7x6dMstWLn2ktjcoiNLDkAGG2EjveHL+Vvc+n0fXumkJYAcSqcVYKtwDU+hDZ0uDUsnUagSYaZrOLAYE8un1A==", + "dev": true, + "engines": { + "node": "^12.0.0 || ^14.0.0 || >=16.0.0" + } + }, "node_modules/@eslint/eslintrc": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.3.3.tgz", - "integrity": "sha512-uj3pT6Mg+3t39fvLrj8iuCIJ38zKO9FpGtJ4BBJebJhEwjoT+KLVNCcHT5QC9NGRIEi7fZ0ZR8YRb884auB4Lg==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.1.0.tgz", + "integrity": "sha512-Lj7DECXqIVCqnqjjHMPna4vn6GJcMgul/wuS0je9OZ9gsL0zzDpKPVtcG1HaDVc+9y+qgXneTeUMbCqXJNpH1A==", "dev": true, "dependencies": { "ajv": "^6.12.4", "debug": "^4.3.2", - "espree": "^9.4.0", - "globals": "^13.15.0", + "espree": "^9.6.0", + "globals": "^13.19.0", "ignore": "^5.2.0", "import-fresh": "^3.2.1", "js-yaml": "^4.1.0", @@ -33,15 +66,24 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/@eslint/js": { + "version": "8.44.0", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.44.0.tgz", + "integrity": "sha512-Ag+9YM4ocKQx9AarydN0KY2j0ErMHNIocPDrVo8zAE44xLTjEtz81OdR68/cydGtk6m6jDb5Za3r2useMzYmSw==", + "dev": true, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, "node_modules/@humanwhocodes/config-array": { - "version": "0.11.6", - "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.6.tgz", - "integrity": "sha512-jJr+hPTJYKyDILJfhNSHsjiwXYf26Flsz8DvNndOsHs5pwSnpGUEy8yzF0JYhCEvTDdV2vuOK5tt8BVhwO5/hg==", + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.10.tgz", + "integrity": "sha512-KVVjQmNUepDVGXNuoRRdmmEjruj0KfiGSbS8LVc12LMsWDQzRXJ0qdhN8L8uUigKpfEHRhlaQFY0ib1tnUbNeQ==", "dev": true, "dependencies": { "@humanwhocodes/object-schema": "^1.2.1", "debug": "^4.1.1", - "minimatch": "^3.0.4" + "minimatch": "^3.0.5" }, "engines": { "node": ">=10.10.0" @@ -102,9 +144,9 @@ } }, "node_modules/acorn": { - "version": "8.8.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.1.tgz", - "integrity": "sha512-7zFpHzhnqYKrkYdUjF1HI1bzd0VygEGX8lFk4k5zVMqHEoES+P+7TKI+EvLO9WVMJ8eekdO0aDEK044xTXwPPA==", + "version": "8.10.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.10.0.tgz", + "integrity": "sha512-F0SAmZ8iUtS//m8DmCTA0jlh6TDKkHQyK6xc6V4KDTyZKA9dnvX9/3sRTVQrWm79glUAZbnmmNcdYwUIHWVybw==", "dev": true, "bin": { "acorn": "bin/acorn" @@ -295,13 +337,16 @@ } }, "node_modules/eslint": { - "version": "8.26.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.26.0.tgz", - "integrity": "sha512-kzJkpaw1Bfwheq4VXUezFriD1GxszX6dUekM7Z3aC2o4hju+tsR/XyTC3RcoSD7jmy9VkPU3+N6YjVU2e96Oyg==", + "version": "8.45.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.45.0.tgz", + "integrity": "sha512-pd8KSxiQpdYRfYa9Wufvdoct3ZPQQuVuU5O6scNgMuOMYuxvH0IGaYK0wUFjo4UYYQQCUndlXiMbnxopwvvTiw==", "dev": true, "dependencies": { - "@eslint/eslintrc": "^1.3.3", - "@humanwhocodes/config-array": "^0.11.6", + "@eslint-community/eslint-utils": "^4.2.0", + "@eslint-community/regexpp": "^4.4.0", + "@eslint/eslintrc": "^2.1.0", + "@eslint/js": "8.44.0", + "@humanwhocodes/config-array": "^0.11.10", "@humanwhocodes/module-importer": "^1.0.1", "@nodelib/fs.walk": "^1.2.8", "ajv": "^6.10.0", @@ -310,34 +355,29 @@ "debug": "^4.3.2", "doctrine": "^3.0.0", "escape-string-regexp": "^4.0.0", - "eslint-scope": "^7.1.1", - "eslint-utils": "^3.0.0", - "eslint-visitor-keys": "^3.3.0", - "espree": "^9.4.0", - "esquery": "^1.4.0", + "eslint-scope": "^7.2.0", + "eslint-visitor-keys": "^3.4.1", + "espree": "^9.6.0", + "esquery": "^1.4.2", "esutils": "^2.0.2", "fast-deep-equal": "^3.1.3", "file-entry-cache": "^6.0.1", "find-up": "^5.0.0", "glob-parent": "^6.0.2", - "globals": "^13.15.0", - "grapheme-splitter": "^1.0.4", + "globals": "^13.19.0", + "graphemer": "^1.4.0", "ignore": "^5.2.0", - "import-fresh": "^3.0.0", "imurmurhash": "^0.1.4", "is-glob": "^4.0.0", "is-path-inside": "^3.0.3", - "js-sdsl": "^4.1.4", "js-yaml": "^4.1.0", "json-stable-stringify-without-jsonify": "^1.0.1", "levn": "^0.4.1", "lodash.merge": "^4.6.2", "minimatch": "^3.1.2", "natural-compare": "^1.4.0", - "optionator": "^0.9.1", - "regexpp": "^3.2.0", + "optionator": "^0.9.3", "strip-ansi": "^6.0.1", - "strip-json-comments": "^3.1.0", "text-table": "^0.2.0" }, "bin": { @@ -351,9 +391,9 @@ } }, "node_modules/eslint-config-prettier": { - "version": "8.5.0", - "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-8.5.0.tgz", - "integrity": "sha512-obmWKLUNCnhtQRKc+tmnYuQl0pFU1ibYJQ5BGhTVB08bHe9wC8qUeG7c08dj9XX+AuPj1YSGSQIHl1pnDHZR0Q==", + "version": "8.8.0", + "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-8.8.0.tgz", + "integrity": "sha512-wLbQiFre3tdGgpDv67NQKnJuTlcUVYHas3k+DZCc2U2BadthoEY4B7hLPvAxaqdyOGCzuLfii2fqGph10va7oA==", "dev": true, "bin": { "eslint-config-prettier": "bin/cli.js" @@ -363,9 +403,9 @@ } }, "node_modules/eslint-scope": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.1.1.tgz", - "integrity": "sha512-QKQM/UXpIiHcLqJ5AOyIW7XZmzjkzQXYE54n1++wb0u9V/abW3l9uQnxX8Z5Xd18xyKIMTUAyQ0k1e8pz6LUrw==", + "version": "7.2.1", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.1.tgz", + "integrity": "sha512-CvefSOsDdaYYvxChovdrPo/ZGt8d5lrJWleAc1diXRKhHGiTYEI26cvo8Kle/wGnsizoCJjK73FMg1/IkIwiNA==", "dev": true, "dependencies": { "esrecurse": "^4.3.0", @@ -373,53 +413,32 @@ }, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" - } - }, - "node_modules/eslint-utils": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-3.0.0.tgz", - "integrity": "sha512-uuQC43IGctw68pJA1RgbQS8/NP7rch6Cwd4j3ZBtgo4/8Flj4eGE7ZYSZRN3iq5pVUv6GPdW5Z1RFleo84uLDA==", - "dev": true, - "dependencies": { - "eslint-visitor-keys": "^2.0.0" - }, - "engines": { - "node": "^10.0.0 || ^12.0.0 || >= 14.0.0" }, "funding": { - "url": "https://github.com/sponsors/mysticatea" - }, - "peerDependencies": { - "eslint": ">=5" - } - }, - "node_modules/eslint-utils/node_modules/eslint-visitor-keys": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.1.0.tgz", - "integrity": "sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw==", - "dev": true, - "engines": { - "node": ">=10" + "url": "https://opencollective.com/eslint" } }, "node_modules/eslint-visitor-keys": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.3.0.tgz", - "integrity": "sha512-mQ+suqKJVyeuwGYHAdjMFqjCyfl8+Ldnxuyp3ldiMBFKkvytrXUZWaiPCEav8qDHKty44bD+qV1IP4T+w+xXRA==", + "version": "3.4.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.1.tgz", + "integrity": "sha512-pZnmmLwYzf+kWaM/Qgrvpen51upAktaaiI01nsJD/Yr3lMOdNtq0cxkrrg16w64VtisN6okbs7Q8AfGqj4c9fA==", "dev": true, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" } }, "node_modules/espree": { - "version": "9.4.0", - "resolved": "https://registry.npmjs.org/espree/-/espree-9.4.0.tgz", - "integrity": "sha512-DQmnRpLj7f6TgN/NYb0MTzJXL+vJF9h3pHy4JhCIs3zwcgez8xmGg3sXHcEO97BrmO2OSvCwMdfdlyl+E9KjOw==", + "version": "9.6.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz", + "integrity": "sha512-oruZaFkjorTpF32kDSI5/75ViwGeZginGGy2NoOSg3Q9bnwlnmDm4HLnkl0RE3n+njDXR037aY1+x58Z/zFdwQ==", "dev": true, "dependencies": { - "acorn": "^8.8.0", + "acorn": "^8.9.0", "acorn-jsx": "^5.3.2", - "eslint-visitor-keys": "^3.3.0" + "eslint-visitor-keys": "^3.4.1" }, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" @@ -429,9 +448,9 @@ } }, "node_modules/esquery": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.4.0.tgz", - "integrity": "sha512-cCDispWt5vHHtwMY2YrAQ4ibFkAL8RbH5YGBnZBc90MolvvfkkQcJro/aZiAQUlQ3qgrYS6D6v8Gc5G5CQsc9w==", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.5.0.tgz", + "integrity": "sha512-YQLXUplAwJgCydQ78IMJywZCceoqk1oH01OERdSAJc/7U2AylwjhSCLDEtqwg811idIS/9fIU5GjG73IgjKMVg==", "dev": true, "dependencies": { "estraverse": "^5.1.0" @@ -489,9 +508,9 @@ "dev": true }, "node_modules/fastq": { - "version": "1.13.0", - "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.13.0.tgz", - "integrity": "sha512-YpkpUnK8od0o1hmeSc7UUs/eB/vIPWJYjKck2QKIzAf71Vm1AAQ3EbuZB3g2JIy+pg+ERD0vqI79KyZiB2e2Nw==", + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz", + "integrity": "sha512-wBrocU2LCXXa+lWBt8RoIRD89Fi8OdABODa/kEnyeyjS5aZO5/GNvI5sEINADqP/h8M29UHTHUb53sUu5Ihqdw==", "dev": true, "dependencies": { "reusify": "^1.0.4" @@ -583,9 +602,9 @@ } }, "node_modules/globals": { - "version": "13.17.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-13.17.0.tgz", - "integrity": "sha512-1C+6nQRb1GwGMKm2dH/E7enFAMxGTmGI7/dEdhy/DNelv85w9B72t3uc5frtMNXIbzrarJJ/lTCjcaZwbLJmyw==", + "version": "13.20.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-13.20.0.tgz", + "integrity": "sha512-Qg5QtVkCy/kv3FUSlu4ukeZDVf9ee0iXLAUYX13gbR17bnejFTzr4iS9bY7kwCf1NztRNm1t91fjOiyx4CSwPQ==", "dev": true, "dependencies": { "type-fest": "^0.20.2" @@ -597,10 +616,10 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/grapheme-splitter": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/grapheme-splitter/-/grapheme-splitter-1.0.4.tgz", - "integrity": "sha512-bzh50DW9kTPM00T8y4o8vQg89Di9oLJVLW/KaOGIXJWP/iqCN6WKYkbNOF04vFLJhwcpYUh9ydh/+5vpOqV4YQ==", + "node_modules/graphemer": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/graphemer/-/graphemer-1.4.0.tgz", + "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==", "dev": true }, "node_modules/has-flag": { @@ -613,9 +632,9 @@ } }, "node_modules/ignore": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.0.tgz", - "integrity": "sha512-CmxgYGiEPCLhfLnpPp1MoRmifwEIOgjcHXxOBjv7mY96c+eWScsOP9c112ZyLdWHi0FxHjI+4uVhKYp/gcdRmQ==", + "version": "5.2.4", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.4.tgz", + "integrity": "sha512-MAb38BcSbH0eHNBxn7ql2NH/kX33OkB3lZ1BNdh7ENeRChHTYsTvWrMubiIAMNS2llXEEgZ1MUOBtXChP3kaFQ==", "dev": true, "engines": { "node": ">= 4" @@ -698,12 +717,6 @@ "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", "dev": true }, - "node_modules/js-sdsl": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/js-sdsl/-/js-sdsl-4.1.5.tgz", - "integrity": "sha512-08bOAKweV2NUC1wqTtf3qZlnpOX/R2DU9ikpjOHs0H+ibQv3zpncVQg6um4uYtRtrwIX8M4Nh3ytK4HGlYAq7Q==", - "dev": true - }, "node_modules/js-yaml": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", @@ -796,17 +809,17 @@ } }, "node_modules/optionator": { - "version": "0.9.1", - "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.1.tgz", - "integrity": "sha512-74RlY5FCnhq4jRxVUPKDaRwrVNXMqsGsiW6AJw4XK8hmtm10wC0ypZBLw5IIp85NZMr91+qd1RvvENwg7jjRFw==", + "version": "0.9.3", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.3.tgz", + "integrity": "sha512-JjCoypp+jKn1ttEFExxhetCKeJt9zhAgAve5FXHixTvFDW/5aEktX9bufBKLRRMdU7bNtpLfcGu94B3cdEJgjg==", "dev": true, "dependencies": { + "@aashutoshrathi/word-wrap": "^1.2.3", "deep-is": "^0.1.3", "fast-levenshtein": "^2.0.6", "levn": "^0.4.1", "prelude-ls": "^1.2.1", - "type-check": "^0.4.0", - "word-wrap": "^1.2.3" + "type-check": "^0.4.0" }, "engines": { "node": ">= 0.8.0" @@ -891,9 +904,9 @@ } }, "node_modules/prettier": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.7.1.tgz", - "integrity": "sha512-ujppO+MkdPqoVINuDFDRLClm7D78qbDt0/NR+wp5FqEZOoTNAjPHWj17QRhu7geIHJfcNhRk1XVQmF8Bp3ye+g==", + "version": "2.8.8", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.8.8.tgz", + "integrity": "sha512-tdN8qQGvNjw4CHbY+XXk0JgCXn9QiF21a55rBe5LJAU+kDyC4WQn4+awm2Xfk2lQMk5fKup9XgzTZtGkjBdP9Q==", "dev": true, "bin": { "prettier": "bin-prettier.js" @@ -906,9 +919,9 @@ } }, "node_modules/punycode": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", - "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz", + "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==", "dev": true, "engines": { "node": ">=6" @@ -934,18 +947,6 @@ } ] }, - "node_modules/regexpp": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-3.2.0.tgz", - "integrity": "sha512-pq2bWo9mVD43nbts2wGv17XLiNLya+GklZ8kaDLV2Z08gDCsGpnKn9BFMepvWuHCbyVvY7J5o5+BVvoQbmlJLg==", - "dev": true, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/mysticatea" - } - }, "node_modules/resolve-from": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", @@ -1114,15 +1115,6 @@ "node": ">= 8" } }, - "node_modules/word-wrap": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", - "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -1141,815 +1133,5 @@ "url": "https://github.com/sponsors/sindresorhus" } } - }, - "dependencies": { - "@eslint/eslintrc": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.3.3.tgz", - "integrity": "sha512-uj3pT6Mg+3t39fvLrj8iuCIJ38zKO9FpGtJ4BBJebJhEwjoT+KLVNCcHT5QC9NGRIEi7fZ0ZR8YRb884auB4Lg==", - "dev": true, - "requires": { - "ajv": "^6.12.4", - "debug": "^4.3.2", - "espree": "^9.4.0", - "globals": "^13.15.0", - "ignore": "^5.2.0", - "import-fresh": "^3.2.1", - "js-yaml": "^4.1.0", - "minimatch": "^3.1.2", - "strip-json-comments": "^3.1.1" - } - }, - "@humanwhocodes/config-array": { - "version": "0.11.6", - "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.6.tgz", - "integrity": "sha512-jJr+hPTJYKyDILJfhNSHsjiwXYf26Flsz8DvNndOsHs5pwSnpGUEy8yzF0JYhCEvTDdV2vuOK5tt8BVhwO5/hg==", - "dev": true, - "requires": { - "@humanwhocodes/object-schema": "^1.2.1", - "debug": "^4.1.1", - "minimatch": "^3.0.4" - } - }, - "@humanwhocodes/module-importer": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", - "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", - "dev": true - }, - "@humanwhocodes/object-schema": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz", - "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==", - "dev": true - }, - "@nodelib/fs.scandir": { - "version": "2.1.5", - "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", - "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", - "dev": true, - "requires": { - "@nodelib/fs.stat": "2.0.5", - "run-parallel": "^1.1.9" - } - }, - "@nodelib/fs.stat": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", - "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", - "dev": true - }, - "@nodelib/fs.walk": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", - "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", - "dev": true, - "requires": { - "@nodelib/fs.scandir": "2.1.5", - "fastq": "^1.6.0" - } - }, - "acorn": { - "version": "8.8.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.1.tgz", - "integrity": "sha512-7zFpHzhnqYKrkYdUjF1HI1bzd0VygEGX8lFk4k5zVMqHEoES+P+7TKI+EvLO9WVMJ8eekdO0aDEK044xTXwPPA==", - "dev": true - }, - "acorn-jsx": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", - "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", - "dev": true, - "requires": {} - }, - "ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", - "dev": true, - "requires": { - "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" - } - }, - "ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true - }, - "ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, - "requires": { - "color-convert": "^2.0.1" - } - }, - "argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true - }, - "balanced-match": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true - }, - "brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "dev": true, - "requires": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "callsites": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", - "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", - "dev": true - }, - "chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "dev": true, - "requires": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - } - }, - "color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, - "requires": { - "color-name": "~1.1.4" - } - }, - "color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true - }, - "concat-map": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", - "dev": true - }, - "cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", - "dev": true, - "requires": { - "path-key": "^3.1.0", - "shebang-command": "^2.0.0", - "which": "^2.0.1" - } - }, - "debug": { - "version": "4.3.4", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", - "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", - "dev": true, - "requires": { - "ms": "2.1.2" - } - }, - "deep-is": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", - "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", - "dev": true - }, - "doctrine": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", - "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==", - "dev": true, - "requires": { - "esutils": "^2.0.2" - } - }, - "escape-string-regexp": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", - "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", - "dev": true - }, - "eslint": { - "version": "8.26.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.26.0.tgz", - "integrity": "sha512-kzJkpaw1Bfwheq4VXUezFriD1GxszX6dUekM7Z3aC2o4hju+tsR/XyTC3RcoSD7jmy9VkPU3+N6YjVU2e96Oyg==", - "dev": true, - "requires": { - "@eslint/eslintrc": "^1.3.3", - "@humanwhocodes/config-array": "^0.11.6", - "@humanwhocodes/module-importer": "^1.0.1", - "@nodelib/fs.walk": "^1.2.8", - "ajv": "^6.10.0", - "chalk": "^4.0.0", - "cross-spawn": "^7.0.2", - "debug": "^4.3.2", - "doctrine": "^3.0.0", - "escape-string-regexp": "^4.0.0", - "eslint-scope": "^7.1.1", - "eslint-utils": "^3.0.0", - "eslint-visitor-keys": "^3.3.0", - "espree": "^9.4.0", - "esquery": "^1.4.0", - "esutils": "^2.0.2", - "fast-deep-equal": "^3.1.3", - "file-entry-cache": "^6.0.1", - "find-up": "^5.0.0", - "glob-parent": "^6.0.2", - "globals": "^13.15.0", - "grapheme-splitter": "^1.0.4", - "ignore": "^5.2.0", - "import-fresh": "^3.0.0", - "imurmurhash": "^0.1.4", - "is-glob": "^4.0.0", - "is-path-inside": "^3.0.3", - "js-sdsl": "^4.1.4", - "js-yaml": "^4.1.0", - "json-stable-stringify-without-jsonify": "^1.0.1", - "levn": "^0.4.1", - "lodash.merge": "^4.6.2", - "minimatch": "^3.1.2", - "natural-compare": "^1.4.0", - "optionator": "^0.9.1", - "regexpp": "^3.2.0", - "strip-ansi": "^6.0.1", - "strip-json-comments": "^3.1.0", - "text-table": "^0.2.0" - } - }, - "eslint-config-prettier": { - "version": "8.5.0", - "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-8.5.0.tgz", - "integrity": "sha512-obmWKLUNCnhtQRKc+tmnYuQl0pFU1ibYJQ5BGhTVB08bHe9wC8qUeG7c08dj9XX+AuPj1YSGSQIHl1pnDHZR0Q==", - "dev": true, - "requires": {} - }, - "eslint-scope": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.1.1.tgz", - "integrity": "sha512-QKQM/UXpIiHcLqJ5AOyIW7XZmzjkzQXYE54n1++wb0u9V/abW3l9uQnxX8Z5Xd18xyKIMTUAyQ0k1e8pz6LUrw==", - "dev": true, - "requires": { - "esrecurse": "^4.3.0", - "estraverse": "^5.2.0" - } - }, - "eslint-utils": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-3.0.0.tgz", - "integrity": "sha512-uuQC43IGctw68pJA1RgbQS8/NP7rch6Cwd4j3ZBtgo4/8Flj4eGE7ZYSZRN3iq5pVUv6GPdW5Z1RFleo84uLDA==", - "dev": true, - "requires": { - "eslint-visitor-keys": "^2.0.0" - }, - "dependencies": { - "eslint-visitor-keys": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.1.0.tgz", - "integrity": "sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw==", - "dev": true - } - } - }, - "eslint-visitor-keys": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.3.0.tgz", - "integrity": "sha512-mQ+suqKJVyeuwGYHAdjMFqjCyfl8+Ldnxuyp3ldiMBFKkvytrXUZWaiPCEav8qDHKty44bD+qV1IP4T+w+xXRA==", - "dev": true - }, - "espree": { - "version": "9.4.0", - "resolved": "https://registry.npmjs.org/espree/-/espree-9.4.0.tgz", - "integrity": "sha512-DQmnRpLj7f6TgN/NYb0MTzJXL+vJF9h3pHy4JhCIs3zwcgez8xmGg3sXHcEO97BrmO2OSvCwMdfdlyl+E9KjOw==", - "dev": true, - "requires": { - "acorn": "^8.8.0", - "acorn-jsx": "^5.3.2", - "eslint-visitor-keys": "^3.3.0" - } - }, - "esquery": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.4.0.tgz", - "integrity": "sha512-cCDispWt5vHHtwMY2YrAQ4ibFkAL8RbH5YGBnZBc90MolvvfkkQcJro/aZiAQUlQ3qgrYS6D6v8Gc5G5CQsc9w==", - "dev": true, - "requires": { - "estraverse": "^5.1.0" - } - }, - "esrecurse": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", - "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", - "dev": true, - "requires": { - "estraverse": "^5.2.0" - } - }, - "estraverse": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", - "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", - "dev": true - }, - "esutils": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", - "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", - "dev": true - }, - "fast-deep-equal": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", - "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "dev": true - }, - "fast-json-stable-stringify": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", - "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", - "dev": true - }, - "fast-levenshtein": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", - "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", - "dev": true - }, - "fastq": { - "version": "1.13.0", - "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.13.0.tgz", - "integrity": "sha512-YpkpUnK8od0o1hmeSc7UUs/eB/vIPWJYjKck2QKIzAf71Vm1AAQ3EbuZB3g2JIy+pg+ERD0vqI79KyZiB2e2Nw==", - "dev": true, - "requires": { - "reusify": "^1.0.4" - } - }, - "file-entry-cache": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", - "integrity": "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==", - "dev": true, - "requires": { - "flat-cache": "^3.0.4" - } - }, - "find-up": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", - "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", - "dev": true, - "requires": { - "locate-path": "^6.0.0", - "path-exists": "^4.0.0" - } - }, - "flat-cache": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.0.4.tgz", - "integrity": "sha512-dm9s5Pw7Jc0GvMYbshN6zchCA9RgQlzzEZX3vylR9IqFfS8XciblUXOKfW6SiuJ0e13eDYZoZV5wdrev7P3Nwg==", - "dev": true, - "requires": { - "flatted": "^3.1.0", - "rimraf": "^3.0.2" - } - }, - "flatted": { - "version": "3.2.7", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.7.tgz", - "integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==", - "dev": true - }, - "fs.realpath": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", - "dev": true - }, - "glob": { - "version": "7.2.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", - "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", - "dev": true, - "requires": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.1.1", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - } - }, - "glob-parent": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", - "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", - "dev": true, - "requires": { - "is-glob": "^4.0.3" - } - }, - "globals": { - "version": "13.17.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-13.17.0.tgz", - "integrity": "sha512-1C+6nQRb1GwGMKm2dH/E7enFAMxGTmGI7/dEdhy/DNelv85w9B72t3uc5frtMNXIbzrarJJ/lTCjcaZwbLJmyw==", - "dev": true, - "requires": { - "type-fest": "^0.20.2" - } - }, - "grapheme-splitter": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/grapheme-splitter/-/grapheme-splitter-1.0.4.tgz", - "integrity": "sha512-bzh50DW9kTPM00T8y4o8vQg89Di9oLJVLW/KaOGIXJWP/iqCN6WKYkbNOF04vFLJhwcpYUh9ydh/+5vpOqV4YQ==", - "dev": true - }, - "has-flag": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "dev": true - }, - "ignore": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.0.tgz", - "integrity": "sha512-CmxgYGiEPCLhfLnpPp1MoRmifwEIOgjcHXxOBjv7mY96c+eWScsOP9c112ZyLdWHi0FxHjI+4uVhKYp/gcdRmQ==", - "dev": true - }, - "import-fresh": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", - "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==", - "dev": true, - "requires": { - "parent-module": "^1.0.0", - "resolve-from": "^4.0.0" - } - }, - "imurmurhash": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", - "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", - "dev": true - }, - "inflight": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", - "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", - "dev": true, - "requires": { - "once": "^1.3.0", - "wrappy": "1" - } - }, - "inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true - }, - "is-extglob": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", - "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", - "dev": true - }, - "is-glob": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", - "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", - "dev": true, - "requires": { - "is-extglob": "^2.1.1" - } - }, - "is-path-inside": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz", - "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==", - "dev": true - }, - "isexe": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", - "dev": true - }, - "js-sdsl": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/js-sdsl/-/js-sdsl-4.1.5.tgz", - "integrity": "sha512-08bOAKweV2NUC1wqTtf3qZlnpOX/R2DU9ikpjOHs0H+ibQv3zpncVQg6um4uYtRtrwIX8M4Nh3ytK4HGlYAq7Q==", - "dev": true - }, - "js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", - "dev": true, - "requires": { - "argparse": "^2.0.1" - } - }, - "json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", - "dev": true - }, - "json-stable-stringify-without-jsonify": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", - "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", - "dev": true - }, - "levn": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", - "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", - "dev": true, - "requires": { - "prelude-ls": "^1.2.1", - "type-check": "~0.4.0" - } - }, - "locate-path": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", - "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", - "dev": true, - "requires": { - "p-locate": "^5.0.0" - } - }, - "lodash.merge": { - "version": "4.6.2", - "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", - "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", - "dev": true - }, - "minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, - "requires": { - "brace-expansion": "^1.1.7" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - }, - "natural-compare": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", - "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", - "dev": true - }, - "once": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "dev": true, - "requires": { - "wrappy": "1" - } - }, - "optionator": { - "version": "0.9.1", - "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.1.tgz", - "integrity": "sha512-74RlY5FCnhq4jRxVUPKDaRwrVNXMqsGsiW6AJw4XK8hmtm10wC0ypZBLw5IIp85NZMr91+qd1RvvENwg7jjRFw==", - "dev": true, - "requires": { - "deep-is": "^0.1.3", - "fast-levenshtein": "^2.0.6", - "levn": "^0.4.1", - "prelude-ls": "^1.2.1", - "type-check": "^0.4.0", - "word-wrap": "^1.2.3" - } - }, - "p-limit": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", - "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", - "dev": true, - "requires": { - "yocto-queue": "^0.1.0" - } - }, - "p-locate": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", - "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", - "dev": true, - "requires": { - "p-limit": "^3.0.2" - } - }, - "parent-module": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", - "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", - "dev": true, - "requires": { - "callsites": "^3.0.0" - } - }, - "path-exists": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", - "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", - "dev": true - }, - "path-is-absolute": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", - "dev": true - }, - "path-key": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", - "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", - "dev": true - }, - "prelude-ls": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", - "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", - "dev": true - }, - "prettier": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.7.1.tgz", - "integrity": "sha512-ujppO+MkdPqoVINuDFDRLClm7D78qbDt0/NR+wp5FqEZOoTNAjPHWj17QRhu7geIHJfcNhRk1XVQmF8Bp3ye+g==", - "dev": true - }, - "punycode": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", - "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", - "dev": true - }, - "queue-microtask": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", - "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", - "dev": true - }, - "regexpp": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-3.2.0.tgz", - "integrity": "sha512-pq2bWo9mVD43nbts2wGv17XLiNLya+GklZ8kaDLV2Z08gDCsGpnKn9BFMepvWuHCbyVvY7J5o5+BVvoQbmlJLg==", - "dev": true - }, - "resolve-from": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", - "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", - "dev": true - }, - "reusify": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", - "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", - "dev": true - }, - "rimraf": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", - "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", - "dev": true, - "requires": { - "glob": "^7.1.3" - } - }, - "run-parallel": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", - "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", - "dev": true, - "requires": { - "queue-microtask": "^1.2.2" - } - }, - "shebang-command": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", - "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "dev": true, - "requires": { - "shebang-regex": "^3.0.0" - } - }, - "shebang-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", - "dev": true - }, - "strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "requires": { - "ansi-regex": "^5.0.1" - } - }, - "strip-json-comments": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", - "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", - "dev": true - }, - "supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dev": true, - "requires": { - "has-flag": "^4.0.0" - } - }, - "text-table": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", - "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", - "dev": true - }, - "type-check": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", - "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", - "dev": true, - "requires": { - "prelude-ls": "^1.2.1" - } - }, - "type-fest": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", - "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", - "dev": true - }, - "uri-js": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", - "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", - "dev": true, - "requires": { - "punycode": "^2.1.0" - } - }, - "which": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", - "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "dev": true, - "requires": { - "isexe": "^2.0.0" - } - }, - "word-wrap": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", - "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==", - "dev": true - }, - "wrappy": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "dev": true - }, - "yocto-queue": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", - "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", - "dev": true - } } } From 4c681d813d77c8aa615b65175f480b9b14bbf63f Mon Sep 17 00:00:00 2001 From: Simon Date: Tue, 25 Jul 2023 00:05:43 +0700 Subject: [PATCH 37/38] fix lint --- tubearchivist/static/script.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tubearchivist/static/script.js b/tubearchivist/static/script.js index cc77074..6ac8f7a 100644 --- a/tubearchivist/static/script.js +++ b/tubearchivist/static/script.js @@ -160,12 +160,12 @@ function dlPending() { }, 500); } -function addToQueue(autostart=false) { +function addToQueue(autostart = false) { let textArea = document.getElementById('id_vid_url'); if (textArea.value === '') { - return + return; } - let toPost = {data: [{youtube_id: textArea.value, status: 'pending'}]}; + let toPost = { data: [{ youtube_id: textArea.value, status: 'pending' }] }; let apiEndpoint = '/api/download/'; if (autostart) { apiEndpoint = `${apiEndpoint}?autostart=true`; From d500fa5eebb015149aac7aa0a577a35650ef225e Mon Sep 17 00:00:00 2001 From: Simon Date: Tue, 25 Jul 2023 00:07:11 +0700 Subject: [PATCH 38/38] add unstable footer --- tubearchivist/config/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubearchivist/config/settings.py b/tubearchivist/config/settings.py index 2dddc88..aec205a 100644 --- a/tubearchivist/config/settings.py +++ b/tubearchivist/config/settings.py @@ -256,4 +256,4 @@ CORS_ALLOW_HEADERS = list(default_headers) + [ # TA application settings TA_UPSTREAM = "https://github.com/tubearchivist/tubearchivist" -TA_VERSION = "v0.3.6" +TA_VERSION = "v0.3.7-unstable"