diff --git a/tubearchivist/home/src/reindex.py b/tubearchivist/home/src/reindex.py index ebaf3c5..dac27e1 100644 --- a/tubearchivist/home/src/reindex.py +++ b/tubearchivist/home/src/reindex.py @@ -297,6 +297,7 @@ class FilesystemScanner: """rename media files as identified by find_bad_media_url""" for bad_filename in self.to_rename: channel, filename, expected_filename = bad_filename + print(f"renaming [{filename}] to [{expected_filename}]") old_path = os.path.join(self.VIDEOS, channel, filename) new_path = os.path.join(self.VIDEOS, channel, expected_filename) os.rename(old_path, new_path) @@ -306,6 +307,7 @@ class FilesystemScanner: bulk_list = [] for video_mismatch in self.mismatch: youtube_id, media_url = video_mismatch + print(f"{youtube_id}: fixing media url {media_url}") action = {"update": {"_id": youtube_id, "_index": "ta_video"}} source = {"doc": {"media_url": media_url}} bulk_list.append(json.dumps(action)) @@ -323,7 +325,8 @@ class FilesystemScanner: def delete_from_index(self): """find indexed but deleted mediafile""" for indexed in self.to_delete: - youtube_id, _ = indexed + youtube_id = indexed[0] + print(f"deleting {youtube_id} from index") url = self.ES_URL + "/ta_video/_doc/" + youtube_id request = requests.delete(url) if not request.ok: @@ -456,12 +459,16 @@ def scan_filesystem(): filesystem_handler = FilesystemScanner() filesystem_handler.list_comarison() if filesystem_handler.to_rename: + print("renaming files") filesystem_handler.rename_files() if filesystem_handler.mismatch: + print("fixing media urls in index") filesystem_handler.send_mismatch_bulk() if filesystem_handler.to_delete: + print("delete metadata from index") filesystem_handler.delete_from_index() if filesystem_handler.to_index: + print("index new videos") for missing_vid in filesystem_handler.to_index: youtube_id = missing_vid[2] index_new_video(youtube_id, missing_vid=missing_vid) diff --git a/tubearchivist/home/tasks.py b/tubearchivist/home/tasks.py index a56c1ad..76ab9b3 100644 --- a/tubearchivist/home/tasks.py +++ b/tubearchivist/home/tasks.py @@ -11,7 +11,11 @@ from home.src.config import AppConfig from home.src.download import ChannelSubscription, PendingList, VideoDownloader from home.src.helper import RedisArchivist, RedisQueue from home.src.index_management import backup_all_indexes, restore_from_backup -from home.src.reindex import ManualImport, reindex_old_documents +from home.src.reindex import ( + ManualImport, + reindex_old_documents, + scan_filesystem +) CONFIG = AppConfig().config REDIS_HOST = os.environ.get("REDIS_HOST") @@ -154,3 +158,9 @@ def kill_dl(task_id): "message": "", } RedisArchivist().set_message("progress:download", mess_dict) + + +@shared_task +def rescan_filesystem(): + """check the media folder for missmatches""" + scan_filesystem() diff --git a/tubearchivist/home/templates/home/settings.html b/tubearchivist/home/templates/home/settings.html index b3e3cb6..1496761 100644 --- a/tubearchivist/home/templates/home/settings.html +++ b/tubearchivist/home/templates/home/settings.html @@ -126,7 +126,11 @@
-

Rescan filesystem.

- Coming soon +

Rescan filesystem

+

Danger Zone: This will delete the metadata of deleted videos from the filesystem.

+

Rescan your media folder looking for missing videos and clean up index. More infos on the Github Wiki.

+
+ +
{% endblock content %} diff --git a/tubearchivist/home/views.py b/tubearchivist/home/views.py index afa8098..8d5cf4f 100644 --- a/tubearchivist/home/views.py +++ b/tubearchivist/home/views.py @@ -22,6 +22,7 @@ from home.tasks import ( download_single, extrac_dl, kill_dl, + rescan_filesystem, run_backup, run_manual_import, run_restore_backup, @@ -506,6 +507,7 @@ class PostData: "manual-import": self.manual_import, "db-backup": self.db_backup, "db-restore": self.db_restore, + "fs-rescan": self.fs_rescan, "channel-search": self.channel_search, } @@ -658,6 +660,13 @@ class PostData: run_restore_backup.delay() return {"success": True} + @staticmethod + def fs_rescan(): + """start file system rescan task""" + print("start filesystem scan") + rescan_filesystem.delay() + return {"success": True} + def channel_search(self): """search for channel name as_you_type""" search_query = self.exec_val diff --git a/tubearchivist/static/script.js b/tubearchivist/static/script.js index 8b916a6..488e47e 100644 --- a/tubearchivist/static/script.js +++ b/tubearchivist/static/script.js @@ -156,7 +156,7 @@ function dbBackup() { function dbRestore() { var payload = JSON.stringify({'db-restore': true}); - sendPost(payload) + sendPost(payload); // clear button var message = document.createElement('p'); message.innerText = 'restoring from backup'; @@ -165,6 +165,17 @@ function dbRestore() { toReplace.appendChild(message); } +function fsRescan() { + var payload = JSON.stringify({'fs-rescan': true}); + sendPost(payload); + // clear button + var message = document.createElement('p'); + message.innerText = 'File system scan in progress'; + var toReplace = document.getElementById('fs-rescan'); + toReplace.innerHTML = ''; + toReplace.appendChild(message); +} + // player function createPlayer(button) { var mediaUrl = button.getAttribute('data-src');