System snapshots, #build

Changed:
- Added: Dedublicated snapshots, read docs/Settings#snapshots first
- Added: Actions for better Python and JS linting
- Changed: Clean up JS
- Changed: Use patched ffmpeg builds for arm64
- API: Added endpoints to interact with snapshots
- Fixed: mobile layout for channel filter dopdown on downloads

.eslintrc.js

@@ -0,0 +1,17 @@
+'use strict';
+module.exports = {
+  extends: ['eslint:recommended', 'eslint-config-prettier'],
+  parserOptions: {
+    ecmaVersion: 2020,
+  },
+  env: {
+    browser: true,
+  },
+  rules: {
+    strict: ['error', 'global'],
+    'no-unused-vars': ['error', { vars: 'local' }],
+    eqeqeq: ['error', 'always', { null: 'ignore' }],
+    curly: ['error', 'multi-line'],
+    'no-var': 'error',
+  },
+};

.github/workflows/lint_js.yml

@@ -0,0 +1,16 @@
+name: lint_js
+
+on: [pull_request, push]
+
+jobs:
+  check:
+    name: lint_js
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '16'
+      - run: npm ci
+      - run: npm run lint
+      - run: npm run format -- --check

.github/workflows/lint_python.yml

@@ -4,16 +4,21 @@ jobs:
   lint_python:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      # note: this logic is duplicated in the `validate` function in ./deploy.sh
+      # if you update this file, you should update that as well
       - run: pip install --upgrade pip wheel
       - run: pip install bandit black codespell flake8 flake8-bugbear
-                         flake8-comprehensions isort
+                         flake8-comprehensions isort requests
-      - run: black --check --diff --line-length 79 .
+      - run: ./deploy.sh validate
-      - run: codespell
+      # - run: black --check --diff --line-length 79 .
-      - run: flake8 . --count --max-complexity=10 --max-line-length=79
+      # - run: codespell --skip="./.git,./package.json,./package-lock.json,./node_modules"
-                      --show-source --statistics
+      # - run: flake8 . --count --max-complexity=10 --max-line-length=79
-      - run: isort --check-only --line-length 79 --profile black .
+      #                 --show-source --statistics
+      # - run: isort --check-only --line-length 79 --profile black .
       # - run: pip install -r tubearchivist/requirements.txt
       # - run: mkdir --parents --verbose .mypy_cache
       # - run: mypy --ignore-missing-imports --install-types --non-interactive .

.gitignore

@@ -5,4 +5,7 @@ __pycache__
 db.sqlite3
 
 # vscode custom conf
-.vscode
+.vscode
+
+# JavaScript stuff
+node_modules

CONTRIBUTING.md

@@ -76,6 +76,10 @@ Do you see anything on the roadmap that you would like to take a closer look at
 
 To fix a bug or implement a feature, fork the repository and make all changes to the testing branch. When ready, create a pull request.
 
+## Making changes to the JavaScript
+
+The JavaScript does not require any build step; you just edit the files directly. However, there is config for eslint and prettier (a linter and formatter respectively); their use is recommended but not required. To use them, install `node`, run `npm i` from the root directory of this repository to install dependencies, then run `npm run lint` and `npm run format` to run eslint and prettier respectively.
+
 ## Releases
 
 There are three different docker tags:

README.md

@@ -195,7 +195,7 @@ Similar to that, TubeArchivist will become all sorts of messed up when running o
 ## Getting Started
 1. Go through the **settings** page and look at the available options. Particularly set *Download Format* to your desired video quality before downloading. **Tube Archivist** downloads the best available quality by default. To support iOS or MacOS and some other browsers a compatible format must be specified. For example:
 ```
-bestvideo[VCODEC=avc1]+bestaudio[ACODEC=mp4a]/mp4
+bestvideo[vcodec*=avc1]+bestaudio[acodec*=mp4a]/mp4
 ```
 2. Subscribe to some of your favorite YouTube channels on the **channels** page. 
 3. On the **downloads** page, click on *Rescan subscriptions* to add videos from the subscribed channels to your Download queue or click on *Add to download queue* to manually add Video IDs, links, channels or playlists.

deploy.sh

@@ -82,10 +82,12 @@ function validate {
 
     echo "run validate on $check_path"
 
+    # note: this logic is duplicated in the `./github/workflows/lint_python.yml` config
+    # if you update this file, you should update that as well
     echo "running black"
     black --diff --color --check -l 79 "$check_path"
     echo "running codespell"
-    codespell --skip="./.git" "$check_path"
+    codespell --skip="./.git,./package.json,./package-lock.json,./node_modules" "$check_path"
     echo "running flake8"
     flake8 "$check_path" --count --max-complexity=10 --max-line-length=79 \
         --show-source --statistics

docs/Settings.md

@@ -72,6 +72,15 @@ Requirements:
     You need the following headers: Content-Type, Accept-Encoding, and Range. Note that the last two headers, Accept-Encoding and Range, are additional headers that you may not have needed previously.  
     Wildcards "*" cannot be used for the Access-Control-Allow-Origin header. If the page has protected media content, it must use a domain instead of a wildcard.
 
+## Snapshots
+System snapshots will automatically make daily snapshots of the Elasticsearch index. Snapshots are deduplicated, meaning that each snapshot will only have to backup changes since the last snapshot. There is also a cleanup function implemented, that will remove snapshots older than 30 days. Due to this improvements compared to our previous solution, system snapshots will replace the current backup system in a future version.
+
+Before activating system snapshots, you'll have to add two additional environment variables to the *archivist-es* container:
+```
+path.repo=/usr/share/elasticsearch/data/snapshot
+TZ=America/New_York
+```
+The variable `path.repo` will set folder where the snapshots will go inside the Elasticsearch container, you can't change the folder, but the variable needs to be set. For the `TZ` variable, set the same as you have for the Tube Archivist container. Rebuild the container for changes to take effect, e.g `docker compose up -d`.
 
 # Scheduler Setup
 Schedule settings expect a cron like format, where the first value is minute, second is hour and third is day of the week. Day 0 is Sunday, day 1 is Monday etc.

package.json

@@ -0,0 +1,17 @@
+{
+  "private": true,
+  "scripts": {
+    "lint": "eslint 'tubearchivist/static/**/*.js'",
+    "format": "prettier --write 'tubearchivist/static/**/*.js'"
+  },
+  "devDependencies": {
+    "eslint": "^8.26.0",
+    "prettier": "^2.7.1",
+    "eslint-config-prettier": "^8.5.0"
+  },
+  "prettier": {
+    "singleQuote": true,
+    "arrowParens": "avoid",
+    "printWidth": 100
+  }
+}

tubearchivist/api/README.md

@@ -29,6 +29,10 @@ Note:
 - [Download Queue List](#download-queue-list-view)
 - [Download Queue Single](#download-queue-item-view)
 
+**Snapshot management**
+- [Snapshot List](#snapshot-list-view)
+- [Snapshot Single](#snapshot-item-view)
+
 **Additional**
 - [Login](#login-view)
 - [Task](#task-view) WIP
@@ -207,6 +211,48 @@ Add to queue previously ignored video:
 DELETE /api/download/\<video_id>/  
 Forget or delete from download queue
 
+## Snapshot List View
+GET /api/snapshot/  
+Return snapshot config and a list of available snapshots.
+
+```json
+{
+    "next_exec": epoch,
+    "next_exec_str": "date_str",
+    "expire_after": "30d",
+    "snapshots": []
+}
+```
+
+POST /api/snapshot/  
+Create new snapshot now, will return immediately, task will run async in the background, will return snapshot name: 
+```json
+{
+    "snapshot_name": "ta_daily_<random-id>
+}
+```
+
+## Snapshot Item View
+GET /api/snapshot/\<snapshot-id>/  
+Return metadata of a single snapshot
+```json
+{
+    "id": "ta_daily_<random-id>,
+    "state": "SUCCESS",
+    "es_version": "0.0.0",
+    "start_date": "date_str",
+    "end_date": "date_str",
+    "end_stamp": epoch,
+    "duration_s": 0
+}
+```
+
+GET /api/snapshot/\<snapshot-id>/  
+Restore this snapshot
+
+DELETE /api/snapshot/\<snapshot-id>/  
+Remove this snapshot from index
+
 ## Login View
 Return token and user ID for username and password:  
 POST /api/login

tubearchivist/api/urls.py

@@ -13,6 +13,8 @@ from api.views import (
     PlaylistApiVideoView,
     PlaylistApiView,
     SearchView,
+    SnapshotApiListView,
+    SnapshotApiView,
     TaskApiView,
     VideoApiListView,
     VideoApiView,
@@ -89,6 +91,16 @@ urlpatterns = [
         TaskApiView.as_view(),
         name="api-task",
     ),
+    path(
+        "snapshot/",
+        SnapshotApiListView.as_view(),
+        name="api-snapshot-list",
+    ),
+    path(
+        "snapshot/<slug:snapshot_id>/",
+        SnapshotApiView.as_view(),
+        name="api-snapshot",
+    ),
     path(
         "cookie/",
         CookieView.as_view(),

tubearchivist/api/views.py

@@ -5,6 +5,7 @@ from api.src.task_processor import TaskHandler
 from home.src.download.queue import PendingInteract
 from home.src.download.yt_dlp_base import CookieHandler
 from home.src.es.connect import ElasticWrap
+from home.src.es.snapshot import ElasticSnapshot
 from home.src.frontend.searching import SearchForm
 from home.src.index.generic import Pagination
 from home.src.index.video import SponsorBlock
@@ -485,6 +486,70 @@ class TaskApiView(ApiBaseView):
         return Response(response)
 
 
+class SnapshotApiListView(ApiBaseView):
+    """resolves to /api/snapshot/
+    GET: returns snashot config plus list of existing snapshots
+    POST: take snapshot now
+    """
+
+    @staticmethod
+    def get(request):
+        """handle get request"""
+        # pylint: disable=unused-argument
+        snapshots = ElasticSnapshot().get_snapshot_stats()
+
+        return Response(snapshots)
+
+    @staticmethod
+    def post(request):
+        """take snapshot now with post request"""
+        # pylint: disable=unused-argument
+        response = ElasticSnapshot().take_snapshot_now()
+
+        return Response(response)
+
+
+class SnapshotApiView(ApiBaseView):
+    """resolves to /api/snapshot/<snapshot-id>/
+    GET: return a single snapshot
+    POST: restore snapshot
+    DELETE: delete a snapshot
+    """
+
+    @staticmethod
+    def get(request, snapshot_id):
+        """handle get request"""
+        # pylint: disable=unused-argument
+        snapshot = ElasticSnapshot().get_single_snapshot(snapshot_id)
+
+        if not snapshot:
+            return Response({"message": "snapshot not found"}, status=404)
+
+        return Response(snapshot)
+
+    @staticmethod
+    def post(request, snapshot_id):
+        """restore snapshot with post request"""
+        # pylint: disable=unused-argument
+        response = ElasticSnapshot().restore_all(snapshot_id)
+        if not response:
+            message = {"message": "failed to restore snapshot"}
+            return Response(message, status=400)
+
+        return Response(response)
+
+    @staticmethod
+    def delete(request, snapshot_id):
+        """delete snapshot from index"""
+        # pylint: disable=unused-argument
+        response = ElasticSnapshot().delete_single_snapshot(snapshot_id)
+        if not response:
+            message = {"message": "failed to delete snapshot"}
+            return Response(message, status=400)
+
+        return Response(response)
+
+
 class CookieView(ApiBaseView):
     """resolves to /api/cookie/
     GET: check if cookie is enabled

tubearchivist/home/apps.py

@@ -6,6 +6,7 @@ import sys
 from django.apps import AppConfig
 from home.src.es.connect import ElasticWrap
 from home.src.es.index_setup import index_check
+from home.src.es.snapshot import ElasticSnapshot
 from home.src.ta.config import AppConfig as ArchivistConfig
 from home.src.ta.ta_redis import RedisArchivist
 
@@ -30,6 +31,7 @@ class StartupCheck:
         self.sync_redis_state()
         self.set_redis_conf()
         self.make_folders()
+        self.snapshot_check()
         self.set_has_run()
 
     def get_has_run(self):
@@ -63,10 +65,7 @@ class StartupCheck:
         cache_dir = self.config_handler.config["application"]["cache_dir"]
         for folder in folders:
             folder_path = os.path.join(cache_dir, folder)
-            try:
+            os.makedirs(folder_path, exist_ok=True)
-                os.makedirs(folder_path)
-            except FileExistsError:
-                continue
 
     def release_lock(self):
         """make sure there are no leftover locks set in redis"""
@@ -84,6 +83,14 @@ class StartupCheck:
             if response:
                 print("deleted leftover key from redis: " + lock)
 
+    def snapshot_check(self):
+        """setup snapshot config, create if needed"""
+        active = self.config_handler.config["application"]["enable_snapshot"]
+        if not active:
+            return
+
+        ElasticSnapshot().setup()
+
     def is_invalid(self, version):
         """return true if es version is invalid, false if ok"""
         major, minor = [int(i) for i in version.split(".")[:2]]

tubearchivist/home/config.json

@@ -37,7 +37,8 @@
         "cache_dir": "/cache",
         "videos": "/youtube",
         "colors": "dark",
-        "enable_cast": false
+        "enable_cast": false,
+        "enable_snapshot": false
     },
     "scheduler": {
         "update_subscribed": false,

tubearchivist/home/src/es/snapshot.py

@@ -0,0 +1,254 @@
+"""
+functionality:
+- handle snapshots in ES
+"""
+
+from datetime import datetime
+from os import environ
+from zoneinfo import ZoneInfo
+
+from home.src.es.connect import ElasticWrap
+from home.src.es.index_setup import get_mapping
+
+
+class ElasticSnapshot:
+    """interact with snapshots on ES"""
+
+    REPO = "ta_snapshot"
+    REPO_SETTINGS = {
+        "compress": "true",
+        "chunk_size": "1g",
+        "location": "/usr/share/elasticsearch/data/snapshot",
+    }
+    POLICY = "ta_daily"
+
+    def __init__(self):
+        self.all_indices = self._get_all_indices()
+
+    def _get_all_indices(self):
+        """return all indices names managed by TA"""
+        mapping = get_mapping()
+        all_indices = [f"ta_{i['index_name']}" for i in mapping]
+
+        return all_indices
+
+    def setup(self):
+        """setup the snapshot in ES, create or update if needed"""
+        print("snapshot: run setup")
+        repo_exists = self._check_repo_exists()
+        if not repo_exists:
+            self.create_repo()
+
+        policy_exists = self._check_policy_exists()
+        if not policy_exists:
+            self.create_policy()
+
+        is_outdated = self._needs_startup_snapshot()
+        if is_outdated:
+            _ = self.take_snapshot_now()
+
+    def _check_repo_exists(self):
+        """check if expected repo already exists"""
+        path = f"_snapshot/{self.REPO}"
+        response, statuscode = ElasticWrap(path).get()
+        if statuscode == 200:
+            print(f"snapshot: repo {self.REPO} already created")
+            matching = response[self.REPO]["settings"] == self.REPO_SETTINGS
+            if not matching:
+                print(f"snapshot: update repo settings {self.REPO_SETTINGS}")
+
+            return matching
+
+        print(f"snapshot: setup repo {self.REPO} config {self.REPO_SETTINGS}")
+        return False
+
+    def create_repo(self):
+        """create filesystem repo"""
+        path = f"_snapshot/{self.REPO}"
+        data = {
+            "type": "fs",
+            "settings": self.REPO_SETTINGS,
+        }
+        response, statuscode = ElasticWrap(path).post(data=data)
+        if statuscode == 200:
+            print(f"snapshot: repo setup correctly: {response}")
+
+    def _check_policy_exists(self):
+        """check if snapshot policy is set correctly"""
+        policy = self._get_policy()
+        expected_policy = self._build_policy_data()
+        if not policy:
+            print(f"snapshot: create policy {self.POLICY} {expected_policy}")
+            return False
+
+        if policy["policy"] != expected_policy:
+            print(f"snapshot: update policy settings {expected_policy}")
+            return False
+
+        print("snapshot: policy is set.")
+        return True
+
+    def _get_policy(self):
+        """get policy from es"""
+        path = f"_slm/policy/{self.POLICY}"
+        response, statuscode = ElasticWrap(path).get()
+        if statuscode != 200:
+            return False
+
+        return response[self.POLICY]
+
+    def create_policy(self):
+        """create snapshot lifetime policy"""
+        path = f"_slm/policy/{self.POLICY}"
+        data = self._build_policy_data()
+        response, statuscode = ElasticWrap(path).put(data)
+        if statuscode == 200:
+            print(f"snapshot: policy setup correctly: {response}")
+
+    def _build_policy_data(self):
+        """build policy dict from config"""
+        return {
+            "schedule": "0 30 1 * * ?",
+            "name": f"<{self.POLICY}_>",
+            "repository": self.REPO,
+            "config": {
+                "indices": self.all_indices,
+                "include_global_state": True,
+            },
+            "retention": {
+                "expire_after": "30d",
+                "min_count": 5,
+                "max_count": 50,
+            },
+        }
+
+    def _needs_startup_snapshot(self):
+        """check if last snapshot is expired"""
+        snap_dicts = self._get_all_snapshots()
+        if not snap_dicts:
+            print("snapshot: create initial snapshot")
+            return True
+
+        last_stamp = snap_dicts[0]["end_stamp"]
+        now = int(datetime.now().strftime("%s"))
+        outdated = (now - last_stamp) / 60 / 60 > 24
+        if outdated:
+            print("snapshot: is outdated, create new now")
+
+        print("snapshot: last snapshot is up-to-date")
+        return outdated
+
+    def take_snapshot_now(self):
+        """execute daily snapshot now"""
+        path = f"_slm/policy/{self.POLICY}/_execute"
+        response, statuscode = ElasticWrap(path).post()
+        if statuscode == 200:
+            print(f"snapshot: executing now: {response}")
+
+        return response
+
+    def get_snapshot_stats(self):
+        """get snapshot info for frontend"""
+        snapshot_info = self._build_policy_details()
+        if snapshot_info:
+            snapshot_info.update({"snapshots": self._get_all_snapshots()})
+
+        return snapshot_info
+
+    def get_single_snapshot(self, snapshot_id):
+        """get single snapshot metadata"""
+        path = f"_snapshot/{self.REPO}/{snapshot_id}"
+        response, statuscode = ElasticWrap(path).get()
+        if statuscode == 404:
+            print(f"snapshots: not found: {snapshot_id}")
+            return False
+
+        snapshot = response["snapshots"][0]
+        return self._parse_single_snapshot(snapshot)
+
+    def _get_all_snapshots(self):
+        """get a list of all registered snapshots"""
+        path = f"_snapshot/{self.REPO}/*?sort=start_time&order=desc"
+        response, statuscode = ElasticWrap(path).get()
+        if statuscode == 404:
+            print("snapshots: not configured")
+            return False
+
+        all_snapshots = response["snapshots"]
+        if not all_snapshots:
+            print("snapshots: no snapshots found")
+            return False
+
+        snap_dicts = []
+        for snapshot in all_snapshots:
+            snap_dict = self._parse_single_snapshot(snapshot)
+            snap_dicts.append(snap_dict)
+
+        return snap_dicts
+
+    def _parse_single_snapshot(self, snapshot):
+        """extract relevant metadata from single snapshot"""
+        snap_dict = {
+            "id": snapshot["snapshot"],
+            "state": snapshot["state"],
+            "es_version": snapshot["version"],
+            "start_date": self._date_converter(snapshot["start_time"]),
+            "end_date": self._date_converter(snapshot["end_time"]),
+            "end_stamp": snapshot["end_time_in_millis"] // 1000,
+            "duration_s": snapshot["duration_in_millis"] // 1000,
+        }
+        return snap_dict
+
+    def _build_policy_details(self):
+        """get additional policy details"""
+        policy = self._get_policy()
+        if not policy:
+            return False
+
+        next_exec = policy["next_execution_millis"] // 1000
+        next_exec_date = datetime.fromtimestamp(next_exec)
+        next_exec_str = next_exec_date.strftime("%Y-%m-%d %H:%M")
+        expire_after = policy["policy"]["retention"]["expire_after"]
+        policy_metadata = {
+            "next_exec": next_exec,
+            "next_exec_str": next_exec_str,
+            "expire_after": expire_after,
+        }
+        return policy_metadata
+
+    @staticmethod
+    def _date_converter(date_utc):
+        """convert datetime string"""
+        expected_format = "%Y-%m-%dT%H:%M:%S.%fZ"
+        date = datetime.strptime(date_utc, expected_format)
+        local_datetime = date.replace(tzinfo=ZoneInfo("localtime"))
+        converted = local_datetime.astimezone(ZoneInfo(environ.get("TZ")))
+        converted_str = converted.strftime("%Y-%m-%d %H:%M")
+
+        return converted_str
+
+    def restore_all(self, snapshot_name):
+        """restore snapshot by name"""
+        for index in self.all_indices:
+            _, _ = ElasticWrap(index).delete()
+
+        path = f"_snapshot/{self.REPO}/{snapshot_name}/_restore"
+        data = {"indices": "*"}
+        response, statuscode = ElasticWrap(path).post(data=data)
+        if statuscode == 200:
+            print(f"snapshot: executing now: {response}")
+            return response
+
+        print(f"snapshot: failed to restore, {statuscode} {response}")
+        return False
+
+    def delete_single_snapshot(self, snapshot_id):
+        """delete single snapshot from index"""
+        path = f"_snapshot/{self.REPO}/{snapshot_id}"
+        response, statuscode = ElasticWrap(path).delete()
+        if statuscode == 200:
+            print(f"snapshot: deleting {snapshot_id} {response}")
+            return response
+
+        print(f"snapshot: failed to delete, {statuscode} {response}")
+        return False

tubearchivist/home/src/frontend/forms.py

@@ -74,6 +74,12 @@ class ApplicationSettingsForm(forms.Form):
         ("1", "enable Cast"),
     ]
 
+    SNAPSHOT_CHOICES = [
+        ("", "-- change snapshot settings --"),
+        ("0", "disable system snapshots"),
+        ("1", "enable system snapshots"),
+    ]
+
     SUBTITLE_SOURCE_CHOICES = [
         ("", "-- change subtitle source settings"),
         ("user", "only download user created"),
@@ -124,6 +130,9 @@ class ApplicationSettingsForm(forms.Form):
     application_enable_cast = forms.ChoiceField(
         widget=forms.Select, choices=CAST_CHOICES, required=False
     )
+    application_enable_snapshot = forms.ChoiceField(
+        widget=forms.Select, choices=SNAPSHOT_CHOICES, required=False
+    )
 
 
 class SchedulerSettingsForm(forms.Form):

tubearchivist/home/templates/home/channel_id.html

@@ -47,7 +47,7 @@
     </div>
 </div>
 <div class="boxed-content {% if view_style == "grid" %}boxed-{{ grid_items }}{% endif %}">
-    <div class="view-controls">
+    <div class="view-controls three">
         <div class="toggle">
             <span>Hide watched videos:</span>
             <div class="toggleBox">

tubearchivist/home/templates/home/downloads.html

@@ -28,7 +28,7 @@
             </div>
         </div>
     </div>
-    <div class="view-controls">
+    <div class="view-controls three">
         <div class="toggle">
             <span>Show only ignored videos:</span>
             <div class="toggleBox">
@@ -42,7 +42,6 @@
         </div>
         <div class="view-icons">
             {% if channel_agg_list|length > 1 %}
-                <span>Filter:</span>
                 <select name="channel_filter" id="channel_filter" onchange="channelFilterDownload(this.value)">
                     <option value="all" {% if not channel_filter_id %}selected{% endif %}>all</option>
                     {% for channel in channel_agg_list %}

tubearchivist/home/templates/home/home.html

@@ -45,7 +45,7 @@
     <div class="title-bar">
         <h1>Recent Videos</h1>
     </div>
-    <div class="view-controls">
+    <div class="view-controls three">
         <div class="toggle">
             <span>Hide watched:</span>
             <div class="toggleBox">

tubearchivist/home/templates/home/settings.html

@@ -77,7 +77,7 @@
                 <ul>
                     <li><span class="settings-current">bestvideo[height<=720]+bestaudio/best[height<=720]</span>: best audio and max video height of 720p.</li>
                     <li><span class="settings-current">bestvideo[height<=1080]+bestaudio/best[height<=1080]</span>: best audio and max video height of 1080p.</li>
-                    <li><span class="settings-current">bestvideo[height<=1080][VCODEC=avc1]+bestaudio[ACODEC=mp4a]/mp4</span>: Max 1080p video height with iOS compatible video and audio codecs.</li>
+                    <li><span class="settings-current">bestvideo[height<=1080][VCODEC*=avc1]+bestaudio[ACODEC*=mp4a]/mp4</span>: Max 1080p video height with iOS compatible video and audio codecs.</li>
                     <li><span class="settings-current">0</span>: deactivate and download the best quality possible as decided by yt-dlp.</li>
                 </ul>
                 <i>Make sure your custom format gets merged into a single file. Check out the <a href="https://github.com/yt-dlp/yt-dlp#format-selection" target="_blank">documentation</a> for valid configurations.</i><br>
@@ -153,6 +153,23 @@
                 {{ app_form.application_enable_cast }}
             </div>
         </div>
+        <div class="settings-group">
+            <h2 id="snapshots">Snapshots</h2>
+            <div class="settings-item">
+                <p>Current system snapshot: <span class="settings-current">{{ config.application.enable_snapshot }}</span></p>
+                <i>Automatically create daily deduplicated snapshots of the index, stored in Elasticsearch. Read first before activating: Wiki.</i><br>
+                {{ app_form.application_enable_snapshot }}
+            </div>
+            <div>
+                {% if snapshots %}
+                    <p>Create next snapshot: <span class="settings-current">{{ snapshots.next_exec_str }}</span>, snapshots expire after <span class="settings-current">{{ snapshots.expire_after }}</span></p>
+                    <br>
+                    {% for snapshot in snapshots.snapshots %}
+                        <p><button id="{{ snapshot.id }}" onclick="restoreSnapshot(id)">Restore</button> Snapshot created on: <span class="settings-current">{{ snapshot.start_date }}</span>, took <span class="settings-current">{{ snapshot.duration_s }}s</span> to create.</p>
+                    {% endfor %}
+                {% endif %}
+            </div>
+        </div>
         <button type="submit" name="application-settings">Update Application Configurations</button>
     </form>
     <div class="title-bar">

tubearchivist/home/views.py

@@ -18,6 +18,7 @@ from django.views import View
 from home.src.download.yt_dlp_base import CookieHandler
 from home.src.es.connect import ElasticWrap
 from home.src.es.index_setup import get_available_backups
+from home.src.es.snapshot import ElasticSnapshot
 from home.src.frontend.api_calls import PostData
 from home.src.frontend.forms import (
     AddToQueueForm,
@@ -942,6 +943,7 @@ class SettingsView(View):
         user_form = UserSettingsForm()
         app_form = ApplicationSettingsForm()
         scheduler_form = SchedulerSettingsForm()
+        snapshots = ElasticSnapshot().get_snapshot_stats()
         token = self.get_token(request)
 
         context = {
@@ -953,6 +955,7 @@ class SettingsView(View):
             "user_form": user_form,
             "app_form": app_form,
             "scheduler_form": scheduler_form,
+            "snapshots": snapshots,
             "version": settings.TA_VERSION,
         }
 
@@ -1000,6 +1003,8 @@ class SettingsView(View):
         for config_value, updated_value in updated:
             if config_value == "cookie_import":
                 self.process_cookie(config, updated_value)
+            if config_value == "enable_snapshot":
+                ElasticSnapshot().setup()
 
     def process_cookie(self, config, updated_value):
         """import and validate cookie"""

tubearchivist/requirements.txt

@@ -8,6 +8,6 @@ Pillow==9.2.0
 redis==4.3.4
 requests==2.28.1
 ryd-client==0.0.6
-uWSGI==2.0.20
+uWSGI==2.0.21
 whitenoise==6.2.0
 yt_dlp==2022.10.4

tubearchivist/static/cast-videos.js

@@ -1,148 +1,157 @@
+'use strict';
+
+/* global cast chrome getVideoPlayerVideoId postVideoProgress setProgressBar getVideoPlayer getVideoPlayerWatchStatus watchedThreshold isWatched getVideoData getURL getVideoPlayerCurrentTime */
+
 function initializeCastApi() {
-    cast.framework.CastContext.getInstance().setOptions({
+  cast.framework.CastContext.getInstance().setOptions({
-        receiverApplicationId: chrome.cast.media.DEFAULT_MEDIA_RECEIVER_APP_ID, // Use built in receiver app on cast device, see https://developers.google.com/cast/docs/styled_receiver if you want to be able to add a theme, splash screen or watermark. Has a $5 one time fee.
+    receiverApplicationId: chrome.cast.media.DEFAULT_MEDIA_RECEIVER_APP_ID, // Use built in receiver app on cast device, see https://developers.google.com/cast/docs/styled_receiver if you want to be able to add a theme, splash screen or watermark. Has a $5 one time fee.
-        autoJoinPolicy: chrome.cast.AutoJoinPolicy.ORIGIN_SCOPED
+    autoJoinPolicy: chrome.cast.AutoJoinPolicy.ORIGIN_SCOPED,
-    });
+  });
 
-    var player = new cast.framework.RemotePlayer();
+  let player = new cast.framework.RemotePlayer();
-    var playerController = new cast.framework.RemotePlayerController(player);
+  let playerController = new cast.framework.RemotePlayerController(player);
 
-    // Add event listerner to check if a connection to a cast device is initiated
+  // Add event listerner to check if a connection to a cast device is initiated
-    playerController.addEventListener(
+  playerController.addEventListener(
-        cast.framework.RemotePlayerEventType.IS_CONNECTED_CHANGED, function() { 
+    cast.framework.RemotePlayerEventType.IS_CONNECTED_CHANGED,
-            castConnectionChange(player) 
+    function () {
-        }
+      castConnectionChange(player);
-    );
+    }
-    playerController.addEventListener(
+  );
-        cast.framework.RemotePlayerEventType.CURRENT_TIME_CHANGED, function() { 
+  playerController.addEventListener(
-            castVideoProgress(player) 
+    cast.framework.RemotePlayerEventType.CURRENT_TIME_CHANGED,
-        }
+    function () {
-    );
+      castVideoProgress(player);
-    playerController.addEventListener(
+    }
-        cast.framework.RemotePlayerEventType.IS_PAUSED_CHANGED, function() { 
+  );
-            castVideoPaused(player) 
+  playerController.addEventListener(
-        }
+    cast.framework.RemotePlayerEventType.IS_PAUSED_CHANGED,
-    );
+    function () {
+      castVideoPaused(player);
+    }
+  );
 }
 
-
 function castConnectionChange(player) {
-    // If cast connection is initialized start cast
+  // If cast connection is initialized start cast
-    if (player.isConnected) {
+  if (player.isConnected) {
-        // console.log("Cast Connected.");
+    // console.log("Cast Connected.");
-        castStart();
+    castStart();
-    } else if (!player.isConnected) {
+  } else if (!player.isConnected) {
-        // console.log("Cast Disconnected.");
+    // console.log("Cast Disconnected.");
-    }
+  }
 }
 
 function castVideoProgress(player) {
-    var videoId = getVideoPlayerVideoId();
+  let videoId = getVideoPlayerVideoId();
-    if (player.mediaInfo.contentId.includes(videoId)) {
+  if (player.mediaInfo.contentId.includes(videoId)) {
-        var currentTime = player.currentTime;
+    let currentTime = player.currentTime;
-        var duration = player.duration;
+    let duration = player.duration;
-        if ((currentTime % 10) <= 1.0 && currentTime != 0 && duration != 0) { // Check progress every 10 seconds or else progress is checked a few times a second
+    if (currentTime % 10 <= 1.0 && currentTime !== 0 && duration !== 0) {
-            postVideoProgress(videoId, currentTime);
+      // Check progress every 10 seconds or else progress is checked a few times a second
-            setProgressBar(videoId, currentTime, duration);
+      postVideoProgress(videoId, currentTime);
-            if (!getVideoPlayerWatchStatus()) { // Check if video is already marked as watched
+      setProgressBar(videoId, currentTime, duration);
-                if (watchedThreshold(currentTime, duration)) {
+      if (!getVideoPlayerWatchStatus()) {
-                    isWatched(videoId);
+        // Check if video is already marked as watched
-                }
+        if (watchedThreshold(currentTime, duration)) {
-            }
+          isWatched(videoId);
         }
+      }
     }
+  }
 }
 
 function castVideoPaused(player) {
-    var videoId = getVideoPlayerVideoId();
+  let videoId = getVideoPlayerVideoId();
-    var currentTime = player.currentTime;
+  let currentTime = player.currentTime;
-    var duration = player.duration;
+  let duration = player.duration;
-    if (player.mediaInfo != null) {
+  if (player.mediaInfo != null) {
-        if (player.mediaInfo.contentId.includes(videoId)) {
+    if (player.mediaInfo.contentId.includes(videoId)) {
-            if (currentTime != 0 && duration != 0) {
+      if (currentTime !== 0 && duration !== 0) {
-                postVideoProgress(videoId, currentTime);
+        postVideoProgress(videoId, currentTime);
-            }
+      }
-        }
     }
+  }
 }
 
 function castStart() {
-    var castSession = cast.framework.CastContext.getInstance().getCurrentSession();
+  let castSession = cast.framework.CastContext.getInstance().getCurrentSession();
-    // Check if there is already media playing on the cast target to prevent recasting on page reload or switching to another video page
+  // Check if there is already media playing on the cast target to prevent recasting on page reload or switching to another video page
-    if (!castSession.getMediaSession()) { 
+  if (!castSession.getMediaSession()) {
-        var videoId = getVideoPlayerVideoId();
+    let videoId = getVideoPlayerVideoId();
-        var videoData = getVideoData(videoId);
+    let videoData = getVideoData(videoId);
-        var contentId = getURL() + videoData.data.media_url;
+    let contentId = getURL() + videoData.data.media_url;
-        var contentTitle = videoData.data.title;
+    let contentTitle = videoData.data.title;
-        var contentImage = getURL() + videoData.data.vid_thumb_url;
+    let contentImage = getURL() + videoData.data.vid_thumb_url;
 
-        contentType = 'video/mp4'; // Set content type, only videos right now so it is hard coded
+    let contentType = 'video/mp4'; // Set content type, only videos right now so it is hard coded
-        contentCurrentTime = getVideoPlayerCurrentTime(); // Get video's current position
+    let contentCurrentTime = getVideoPlayerCurrentTime(); // Get video's current position
-        contentActiveSubtitle = [];
+    let contentActiveSubtitle = [];
-        // Check if a subtitle is turned on.
+    // Check if a subtitle is turned on.
-        for (var i = 0; i < getVideoPlayer().textTracks.length; i++) {
+    for (let i = 0; i < getVideoPlayer().textTracks.length; i++) {
-            if (getVideoPlayer().textTracks[i].mode == "showing") {
+      if (getVideoPlayer().textTracks[i].mode === 'showing') {
-                contentActiveSubtitle =[i + 1];
+        contentActiveSubtitle = [i + 1];
-            }
+      }
-        }
+    }
-        contentSubtitles = [];
+    let contentSubtitles = [];
-        var videoSubtitles = videoData.data.subtitles; // Array of subtitles
+    let videoSubtitles = videoData.data.subtitles; // Array of subtitles
-        if (typeof(videoSubtitles) != 'undefined' && videoData.config.downloads.subtitle) {
+    if (typeof videoSubtitles !== 'undefined' && videoData.config.downloads.subtitle) {
-            for (var i = 0; i < videoSubtitles.length; i++) {
+      for (let i = 0; i < videoSubtitles.length; i++) {
-                subtitle = new chrome.cast.media.Track(i, chrome.cast.media.TrackType.TEXT);
+        let subtitle = new chrome.cast.media.Track(i, chrome.cast.media.TrackType.TEXT);
-                subtitle.trackContentId = videoSubtitles[i].media_url;
+        subtitle.trackContentId = videoSubtitles[i].media_url;
-                subtitle.trackContentType = 'text/vtt';
+        subtitle.trackContentType = 'text/vtt';
-                subtitle.subtype = chrome.cast.media.TextTrackType.SUBTITLES;
+        subtitle.subtype = chrome.cast.media.TextTrackType.SUBTITLES;
-                subtitle.name = videoSubtitles[i].name;
+        subtitle.name = videoSubtitles[i].name;
-                subtitle.language = videoSubtitles[i].lang;
+        subtitle.language = videoSubtitles[i].lang;
-                subtitle.customData = null;
+        subtitle.customData = null;
-                contentSubtitles.push(subtitle);
+        contentSubtitles.push(subtitle);
-            }
+      }
-        }
+    }
 
-        mediaInfo = new chrome.cast.media.MediaInfo(contentId, contentType); // Create MediaInfo var that contains url and content type
+    let mediaInfo = new chrome.cast.media.MediaInfo(contentId, contentType); // Create MediaInfo var that contains url and content type
-        // mediaInfo.streamType = chrome.cast.media.StreamType.BUFFERED; // Set type of stream, BUFFERED, LIVE, OTHER
+    // mediaInfo.streamType = chrome.cast.media.StreamType.BUFFERED; // Set type of stream, BUFFERED, LIVE, OTHER
-        mediaInfo.metadata = new chrome.cast.media.GenericMediaMetadata(); // Create metadata var and add it to MediaInfo
+    mediaInfo.metadata = new chrome.cast.media.GenericMediaMetadata(); // Create metadata var and add it to MediaInfo
-        mediaInfo.metadata.title = contentTitle.replace("&amp;", "&"); // Set the video title
+    mediaInfo.metadata.title = contentTitle.replace('&amp;', '&'); // Set the video title
-        mediaInfo.metadata.images = [new chrome.cast.Image(contentImage)]; // Set the video thumbnail
+    mediaInfo.metadata.images = [new chrome.cast.Image(contentImage)]; // Set the video thumbnail
-        // mediaInfo.textTrackStyle = new chrome.cast.media.TextTrackStyle();
+    // mediaInfo.textTrackStyle = new chrome.cast.media.TextTrackStyle();
-        mediaInfo.tracks = contentSubtitles;
+    mediaInfo.tracks = contentSubtitles;
 
-        var request = new chrome.cast.media.LoadRequest(mediaInfo); // Create request with the previously set MediaInfo. 
+    let request = new chrome.cast.media.LoadRequest(mediaInfo); // Create request with the previously set MediaInfo.
-        // request.queueData = new chrome.cast.media.QueueData(); // See https://developers.google.com/cast/docs/reference/web_sender/chrome.cast.media.QueueData for playlist support.
+    // request.queueData = new chrome.cast.media.QueueData(); // See https://developers.google.com/cast/docs/reference/web_sender/chrome.cast.media.QueueData for playlist support.
-        request.currentTime = shiftCurrentTime(contentCurrentTime); // Set video start position based on the browser video position
+    request.currentTime = shiftCurrentTime(contentCurrentTime); // Set video start position based on the browser video position
-        request.activeTrackIds = contentActiveSubtitle; // Set active subtitle based on video player
+    request.activeTrackIds = contentActiveSubtitle; // Set active subtitle based on video player
-        // request.autoplay = false; // Set content to auto play, true by default
+    // request.autoplay = false; // Set content to auto play, true by default
-        castSession.loadMedia(request).then(
+    castSession.loadMedia(request).then(
-            function() { 
+      function () {
-                castSuccessful();
+        castSuccessful();
-            }, 
+      },
-            function() { 
+      function (error) {
-                castFailed(errorCode); 
+        castFailed(error.code);
-            }
+      }
-        ); // Send request to cast device
+    ); // Send request to cast device
-    }   
+  }
 }
 
-function shiftCurrentTime(contentCurrentTime) { // Shift media back 3 seconds to prevent missing some of the content
+function shiftCurrentTime(contentCurrentTime) {
-    if (contentCurrentTime > 5) {
+  // Shift media back 3 seconds to prevent missing some of the content
-        return(contentCurrentTime - 3);
+  if (contentCurrentTime > 5) {
-    } else {
+    return contentCurrentTime - 3;
-        return(0);
+  } else {
-    }
+    return 0;
+  }
 }
 
 function castSuccessful() {
-    // console.log('Cast Successful.');
+  // console.log('Cast Successful.');
-    getVideoPlayer().pause(); // Pause browser video on successful cast
+  getVideoPlayer().pause(); // Pause browser video on successful cast
 }
 
 function castFailed(errorCode) {
-    console.log('Error code: ' + errorCode);
+  console.log('Error code: ' + errorCode);
 }
 
-window['__onGCastApiAvailable'] = function(isAvailable) {
+window['__onGCastApiAvailable'] = function (isAvailable) {
-    if (isAvailable) {
+  if (isAvailable) {
-        initializeCastApi();
+    initializeCastApi();
-    }
+  }
-}
+};

tubearchivist/static/css/style.css

@@ -1182,6 +1182,10 @@ video:-webkit-full-screen {
         margin: 15px;
         text-align: center;
     }
+    .view-controls.three {
+        grid-template-columns: unset;
+        justify-content: center;
+    }
     .sort {
         display: block;
     }

tubearchivist/static/progress.js

@@ -1,106 +1,110 @@
 /**
  * Handle multi channel notifications
- * 
+ *
  */
 
-checkMessages()
+'use strict';
+
+checkMessages();
 
 // page map to notification status
 const messageTypes = {
-    "download": ["message:download", "message:add", "message:rescan", "message:playlistscan"],
+  download: ['message:download', 'message:add', 'message:rescan', 'message:playlistscan'],
-    "channel": ["message:subchannel"],
+  channel: ['message:subchannel'],
-    "channel_id": ["message:playlistscan"],
+  channel_id: ['message:playlistscan'],
-    "playlist": ["message:subplaylist"],
+  playlist: ['message:subplaylist'],
-    "setting": ["message:setting"]
+  setting: ['message:setting'],
-}
+};
 
 // start to look for messages
 function checkMessages() {
-    var notifications = document.getElementById("notifications");
+  let notifications = document.getElementById('notifications');
-    if (notifications) {
+  if (notifications) {
-        var dataOrigin = notifications.getAttribute("data");
+    let dataOrigin = notifications.getAttribute('data');
-        getMessages(dataOrigin);
+    getMessages(dataOrigin);
-    }
+  }
 }
 
 // get messages for page on timer
 function getMessages(dataOrigin) {
-    fetch('/progress/').then(response => {
+  fetch('/progress/')
-        return response.json();
+    .then(response => {
-    }).then(responseData => {
+      return response.json();
-        var messages = buildMessage(responseData, dataOrigin);
+    })
-        if (messages.length > 0) {
+    .then(responseData => {
-            // restart itself
+      let messages = buildMessage(responseData, dataOrigin);
-            setTimeout(function() {
+      if (messages.length > 0) {
-                getMessages(dataOrigin);
+        // restart itself
-            }, 3000);
+        setTimeout(function () {
-        };
+          getMessages(dataOrigin);
+        }, 3000);
+      }
     });
 }
 
 // make div for all messages, return relevant
 function buildMessage(responseData, dataOrigin) {
-    // filter relevan messages
+  // filter relevan messages
-    var allMessages = responseData["messages"];
+  let allMessages = responseData['messages'];
-    var messages = allMessages.filter(function(value) {
+  let messages = allMessages.filter(function (value) {
-        return messageTypes[dataOrigin].includes(value["status"])
+    return messageTypes[dataOrigin].includes(value['status']);
-    }, dataOrigin);
+  }, dataOrigin);
-    // build divs
+  // build divs
-    var notificationDiv = document.getElementById("notifications");
+  let notificationDiv = document.getElementById('notifications');
-    var nots = notificationDiv.childElementCount;
+  let nots = notificationDiv.childElementCount;
-    notificationDiv.innerHTML = "";
+  notificationDiv.innerHTML = '';
-    for (let i = 0; i < messages.length; i++) {
+  for (let i = 0; i < messages.length; i++) {
-        var messageData = messages[i];
+    let messageData = messages[i];
-        var messageStatus = messageData["status"];
+    let messageStatus = messageData['status'];
-        var messageBox = document.createElement("div");
+    let messageBox = document.createElement('div');
-        var title = document.createElement("h3");
+    let title = document.createElement('h3');
-        title.innerHTML = messageData["title"];
+    title.innerHTML = messageData['title'];
-        var message = document.createElement("p");
+    let message = document.createElement('p');
-        message.innerHTML = messageData["message"];
+    message.innerHTML = messageData['message'];
-        messageBox.appendChild(title);
+    messageBox.appendChild(title);
-        messageBox.appendChild(message);
+    messageBox.appendChild(message);
-        messageBox.classList.add(messageData["level"], "notification");
+    messageBox.classList.add(messageData['level'], 'notification');
-        notificationDiv.appendChild(messageBox);
+    notificationDiv.appendChild(messageBox);
-        if (messageStatus === "message:download") {
+    if (messageStatus === 'message:download') {
-            checkDownloadIcons();
+      checkDownloadIcons();
-        };
+    }
-    };
+  }
-    // reload page when no more notifications
+  // reload page when no more notifications
-    if (nots > 0 && messages.length === 0) {
+  if (nots > 0 && messages.length === 0) {
-        location.reload();
+    location.reload();
-    };
+  }
-    return messages
+  return messages;
 }
 
 // check if download icons are needed
 function checkDownloadIcons() {
-    var iconBox = document.getElementById("downloadControl");
+  let iconBox = document.getElementById('downloadControl');
-    if (iconBox.childElementCount === 0) {
+  if (iconBox.childElementCount === 0) {
-        var downloadIcons = buildDownloadIcons();
+    let downloadIcons = buildDownloadIcons();
-        iconBox.appendChild(downloadIcons);
+    iconBox.appendChild(downloadIcons);
-    };
+  }
 }
 
 // add dl control icons
 function buildDownloadIcons() {
-    var downloadIcons = document.createElement('div');
+  let downloadIcons = document.createElement('div');
-    downloadIcons.classList = 'dl-control-icons';
+  downloadIcons.classList = 'dl-control-icons';
-    // stop icon
+  // stop icon
-    var stopIcon = document.createElement('img');
+  let stopIcon = document.createElement('img');
-    stopIcon.setAttribute('id', "stop-icon");
+  stopIcon.setAttribute('id', 'stop-icon');
-    stopIcon.setAttribute('title', "Stop Download Queue");
+  stopIcon.setAttribute('title', 'Stop Download Queue');
-    stopIcon.setAttribute('src', "/static/img/icon-stop.svg");
+  stopIcon.setAttribute('src', '/static/img/icon-stop.svg');
-    stopIcon.setAttribute('alt', "stop icon");
+  stopIcon.setAttribute('alt', 'stop icon');
-    stopIcon.setAttribute('onclick', 'stopQueue()');
+  stopIcon.setAttribute('onclick', 'stopQueue()');
-    // kill icon
+  // kill icon
-    var killIcon = document.createElement('img');
+  let killIcon = document.createElement('img');
-    killIcon.setAttribute('id', "kill-icon");
+  killIcon.setAttribute('id', 'kill-icon');
-    killIcon.setAttribute('title', "Kill Download Queue");
+  killIcon.setAttribute('title', 'Kill Download Queue');
-    killIcon.setAttribute('src', "/static/img/icon-close.svg");
+  killIcon.setAttribute('src', '/static/img/icon-close.svg');
-    killIcon.setAttribute('alt', "kill icon");
+  killIcon.setAttribute('alt', 'kill icon');
-    killIcon.setAttribute('onclick', 'killQueue()');
+  killIcon.setAttribute('onclick', 'killQueue()');
-    // stich together
+  // stich together
-    downloadIcons.appendChild(stopIcon);
+  downloadIcons.appendChild(stopIcon);
-    downloadIcons.appendChild(killIcon);
+  downloadIcons.appendChild(killIcon);
-    return downloadIcons
+  return downloadIcons;
 }