initial commit, MVP

.gitignore

@@ -0,0 +1,9 @@
+# config file
+config.json
+
+# py
+__pycache__
+.mypy_cache
+
+# editor
+.vscode

README.md

@@ -0,0 +1,67 @@
+## Tube Archivist Jellyfin Integration
+Import your Tube Archivist media folder into Jellyfin
+
+![home screenshot](assets/screenshot-home.png?raw=true "Jellyfin Home")
+
+This is a proof of concept, expect bugs, for the time being, only use it for your testing environment, *not* for your main Jellyfin server. This requires Tube Archivist *unstable* builds for API compatibility.
+
+## Core functionality
+- Import each YouTube channel as a TV Show
+- Each year will become a Season of that Show
+- Load artwork and additional metadata into Jellyfin
+
+## How does that work?
+At the core, this links the two APIs together: This first queries the Jellyfin API for YouTube videos the goes to look up the required metadata in the Tube Archivist API. Then as a secondary step this will transfer the artwork as indexed from Tube Archivist.
+
+This doesn't depend on any additional Jellyfin plugins, that is a stand alone solution.
+
+## Setup Jellyfin
+0. Add the Tube Archivist **/youtube** folder as a media folder for Jellyfin.
+    - IMPORTANT: This needs to be mounted as **read only** aka `ro`, otherwise this will mess up Tube Archivist.
+
+Example Jellyfin setup:
+```yml
+jellyfin:
+  image: jellyfin/jellyfin
+  container_name: jellyfin
+  restart: unless-stopped
+  network_mode: host
+  volumes:
+    - ./volume/jellyfin/config:/config
+    - ./volume/jellyfin/cache:/cache
+    - ./volume/tubearchivist/youtube:/media/youtube:ro  # note :ro at the end
+```
+
+1. Add a new media library to your Jellyfin server for your Tube Archivist videos, required options:
+    - Content type: `Shows`
+    - Displayname: `YouTube`
+    - Folder: Root folder for Tube Archivist videos
+    - Deactivate all Metadata downloaders
+    - Automatically refresh metadata from the internet: `Never`
+    - Deactivate all Image fetchers
+
+2. Let Jellyfin complete the library scan
+    - This works best if Jellyfin has found all media files and Tube Archivist isn't currently downloading.
+    - At first, this will add all channels as a Show with a single Season 1.
+    - Then this script will populate the metadata
+
+## Install Standalone
+1. Install required libraries for your environment, e.g.
+```bash
+pip install -r requirements.txt
+```
+2. rename/copy *config.sample.json* to *config.json*.
+3. configure these keys:
+	- `ta_video_path`: Absolute path of your /youtube folder from Tube Archivist
+	- `ta_url`: Full URL where Tube Archivist is reachable
+	- `ta_token`: Tube Archivist API token, accessible from the settings page
+	- `jf_url`: Full URL where Jellyfin is reachable
+	- `jf_token`: Jellyfin API token
+
+Then run the script with python, e.g.
+```python
+python main.py
+```
+
+## Install with Docker
+Coming soon...

config.sample.json

@@ -0,0 +1,7 @@
+{
+    "ta_video_path": "/media/docker/volume/tubearchivist/youtube",
+    "ta_url": "http://tubearchivist.local",
+    "ta_token": "xxxxxxxxxxxxxxxx",
+    "jf_url": "http://jellyfin.local:8096",
+    "jf_token": "yyyyyyyyyyyyyyyy"
+}

deploy.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+set -e
+
+function validate {
+
+    if [[ $1 ]]; then
+        check_path="$1"
+    else
+        check_path="."
+    fi
+
+    echo "run validate on $check_path"
+
+    # note: this logic is duplicated in the `./github/workflows/lint_python.yml` config
+    # if you update this file, you should update that as well
+    echo "running black"
+    black --exclude "migrations/*" --diff --color --check -l 79 "$check_path"
+    echo "running codespell"
+    codespell --skip="./.git,./package.json,./package-lock.json,./node_modules,./.mypy_cache" "$check_path"
+    echo "running flake8"
+    flake8 "$check_path" --exclude "migrations" --count --max-complexity=10 \
+        --max-line-length=79 --show-source --statistics
+    echo "running isort"
+    isort --skip "migrations" --check-only --diff --profile black -l 79 "$check_path"
+    printf "    \n> all validations passed\n"
+
+}
+
+if [[ $1 == "validate" ]]; then
+    validate "$2"
+else
+    echo "valid options are: validate"
+fi
+
+
+##
+exit 0

main.py

@@ -0,0 +1,18 @@
+"""application entry point"""
+
+from src.connect import Jellyfin, TubeArchivist, folder_check
+from src.series import Library
+
+folder_check()
+Jellyfin().ping()
+TubeArchivist().ping()
+
+
+def main():
+    """main thread"""
+    library = Library()
+    library.validate_series()
+
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1 @@
+requests

src/config.py

@@ -0,0 +1,11 @@
+"""handle config file"""
+
+import json
+
+
+def get_config():
+    """get connection config"""
+    with open("config.json", "r", encoding="utf-8") as f:
+        config_content = json.loads(f.read())
+
+    return config_content

src/connect.py

@@ -0,0 +1,104 @@
+"""handle connections"""
+
+import base64
+import os
+
+import requests
+
+from src.config import get_config
+
+CONFIG = get_config()
+
+
+class Jellyfin:
+    """connect to jellyfin"""
+
+    headers = {"Authorization": "MediaBrowser Token=" + CONFIG["jf_token"]}
+    base = CONFIG["jf_url"]
+
+    def get(self, path):
+        """make a get request"""
+        url = f"{self.base}/{path}"
+        response = requests.get(url, headers=self.headers, timeout=10)
+        if response.ok:
+            return response.json()
+
+        print(response.text)
+        return False
+
+    def post(self, path, data):
+        """make a post request"""
+        url = f"{self.base}/{path}"
+        response = requests.post(
+            url, headers=self.headers, json=data, timeout=10
+        )
+        if not response.ok:
+            print(response.text)
+
+    def post_img(self, path, thumb_base64):
+        """set image"""
+        url = f"{self.base}/{path}"
+        new_headers = self.headers.copy()
+        new_headers.update({"Content-Type": "image/jpeg"})
+        response = requests.post(
+            url, headers=new_headers, data=thumb_base64, timeout=10
+        )
+        if not response.ok:
+            print(response.text)
+
+    def ping(self):
+        """ping the server"""
+        path = "Users"
+        response = self.get(path)
+        if not response:
+            raise ConnectionError("failed to connect to jellyfin")
+
+        print("[connection] verified jellyfin connection")
+
+
+class TubeArchivist:
+    """connect to Tube Archivist"""
+
+    headers = {"Authorization": "Token " + CONFIG.get("ta_token")}
+    base = CONFIG["ta_url"]
+
+    def get(self, path):
+        """get document from ta"""
+        url = f"{self.base}/api/{path}"
+        response = requests.get(url, headers=self.headers, timeout=10)
+
+        if response.ok:
+            response_json = response.json()
+            if "data" in response_json:
+                return response.json().get("data")
+
+            return response.json()
+
+        print(response.text)
+        return False
+
+    def get_thumb(self, path):
+        """get encoded thumbnail from ta"""
+        url = CONFIG.get("ta_url") + path
+        response = requests.get(
+            url, headers=self.headers, stream=True, timeout=10
+        )
+
+        return base64.b64encode(response.content)
+
+    def ping(self):
+        """ping tubearchivist server"""
+        response = self.get("ping/")
+        if not response:
+            raise ConnectionError("failed to connect to tube archivist")
+
+        print("[connection] verified tube archivist connection")
+
+
+def folder_check():
+    """check if ta_video_path is accessible"""
+    if not os.path.exists("config.json"):
+        raise FileNotFoundError("config.json file not found")
+
+    if not os.path.exists(CONFIG["ta_video_path"]):
+        raise FileNotFoundError("failed to access ta_video_path")

src/episode.py

@@ -0,0 +1,58 @@
+"""set metadata to episodes"""
+
+from datetime import datetime
+
+from src.connect import Jellyfin, TubeArchivist
+
+
+class Episode:
+    """interact with an single episode"""
+
+    def __init__(self, youtube_id, jf_id):
+        self.youtube_id = youtube_id
+        self.jf_id = jf_id
+
+    def sync(self):
+        """sync episode metadata"""
+        ta_video = self.get_ta_video()
+        self.update_metadata(ta_video)
+        self.update_artwork(ta_video)
+
+    def get_ta_video(self):
+        """get video metadata from ta"""
+        path = f"/video/{self.youtube_id}"
+        ta_video = TubeArchivist().get(path)
+
+        return ta_video
+
+    def update_metadata(self, ta_video):
+        """update jellyfin metadata from item_id"""
+        published = ta_video.get("published")
+        published_date = datetime.strptime(published, "%d %b, %Y")
+        data = {
+            "Id": self.jf_id,
+            "Name": ta_video.get("title"),
+            "Genres": [],
+            "Tags": [],
+            "ProductionYear": published_date.year,
+            "ProviderIds": {},
+            "ParentIndexNumber": published_date.year,
+            "PremiereDate": published_date.isoformat(),
+            "Overview": self._get_desc(ta_video),
+        }
+        path = f"Items/{self.jf_id}"
+        Jellyfin().post(path, data)
+
+    def update_artwork(self, ta_video):
+        """update episode artwork in jf"""
+        thumb_base64 = TubeArchivist().get_thumb(ta_video.get("vid_thumb_url"))
+        path = f"Items/{self.jf_id}/Images/Primary"
+        Jellyfin().post_img(path, thumb_base64)
+
+    def _get_desc(self, ta_video):
+        """get description"""
+        raw_desc = ta_video.get("description").replace("\n", "<br>")
+        if len(raw_desc) > 500:
+            return raw_desc[:500] + " ..."
+
+        return raw_desc

src/season.py

@@ -0,0 +1,5 @@
+"""set metadata to seasons"""
+
+
+def setup_seasons():
+    """setup all missing seasons"""

src/series.py

@@ -0,0 +1,173 @@
+"""set metadata to shows"""
+
+import base64
+import os
+from time import sleep
+
+from src.config import get_config
+from src.connect import Jellyfin, TubeArchivist
+from src.episode import Episode
+
+
+class Library:
+    """grouped series"""
+
+    def __init__(self):
+        self.yt_collection = self.get_yt_collection()
+
+    def get_yt_collection(self):
+        """get collection id for youtube folder"""
+        path = "Items?Recursive=true&includeItemTypes=Folder"
+        folders = Jellyfin().get(path)
+        for folder in folders["Items"]:
+            if folder.get("Name").lower() == "youtube":
+                return folder.get("Id")
+
+        raise ValueError("youtube folder not found")
+
+    def validate_series(self):
+        """validate all series"""
+        all_shows = self._get_all_series()
+        for show in all_shows["Items"]:
+            show_handler = Show(show)
+            folders = show_handler.create_folders()
+            show_handler.validate_show()
+            show_handler.validate_episodes()
+            show_handler.delete_folders(folders)
+
+        self.set_collection_art()
+
+    def _get_all_series(self):
+        """get all shows indexed in jf"""
+        path = f"Items?Recursive=true&IncludeItemTypes=Series&fields=ParentId,Path&ParentId={self.yt_collection}"  # noqa: E501
+        all_shows = Jellyfin().get(path)
+
+        return all_shows
+
+    def set_collection_art(self):
+        """set collection ta art"""
+        with open("assets/collection-art.jpg", "rb") as f:
+            asset = f.read()
+
+        folders = Jellyfin().get("Library/MediaFolders")
+        for folder in folders["Items"]:
+            if folder.get("Name").lower() == "youtube":
+                jf_id = folder.get("Id")
+                path = f"Items/{jf_id}/Images/Primary"
+                Jellyfin().post_img(path, base64.b64encode(asset))
+
+
+class Show:
+    """interact with a single show"""
+
+    def __init__(self, show):
+        self.show = show
+
+    def _get_all_episodes(self):
+        """get all episodes of show"""
+        series_id = self.show.get("Id")
+        path = f"Shows/{series_id}/Episodes?fields=Path"
+        all_episodes = Jellyfin().get(path)
+
+        return all_episodes["Items"]
+
+    def _get_expected_seasons(self):
+        """get all expected seasons"""
+        episodes = self._get_all_episodes()
+        all_years = {os.path.split(i["Path"])[-1][:4] for i in episodes}
+
+        return all_years
+
+    def _get_existing_seasons(self):
+        """get all seasons indexed of series"""
+        series_id = self.show.get("Id")
+        path = f"Shows/{series_id}/Seasons"
+        all_seasons = Jellyfin().get(path)
+
+        return [str(i.get("IndexNumber")) for i in all_seasons["Items"]]
+
+    def create_folders(self):
+        """create season folders if needed"""
+        all_expected = self._get_expected_seasons()
+        all_existing = self._get_existing_seasons()
+
+        base = get_config().get("ta_video_path")
+        channel_name = os.path.split(self.show["Path"])[-1]
+        folders = []
+        for year in all_expected:
+            if year not in all_existing:
+                path = os.path.join(base, channel_name, year)
+                os.mkdir(path)
+                folders.append(path)
+
+        self._wait_for_seasons()
+        return folders
+
+    def delete_folders(self, folders):
+        """delete temporary folders created"""
+        for folder in folders:
+            os.removedirs(folder)
+
+    def _wait_for_seasons(self):
+        """wait for seasons to be created"""
+        jf_id = self.show["Id"]
+        path = f"Items/{jf_id}/Refresh?Recursive=true&ImageRefreshMode=Default&MetadataRefreshMode=Default"  # noqa: E501
+        Jellyfin().post(path, False)
+        for _ in range(12):
+            all_existing = set(self._get_existing_seasons())
+            all_expected = self._get_expected_seasons()
+            if all_expected.issubset(all_existing):
+                return
+
+            print(f"[setup][{jf_id}] waiting for seasons to be created")
+            sleep(5)
+
+        raise TimeoutError("timeout reached for creating season folder")
+
+    def validate_show(self):
+        """set show metadata"""
+        ta_channel = self._get_ta_channel()
+        self.update_metadata(ta_channel)
+        self.update_artwork(ta_channel)
+
+    def _get_ta_channel(self):
+        """get ta channel metadata"""
+        episode = self._get_all_episodes()[0]
+        youtube_id = os.path.split(episode["Path"])[-1][9:20]
+        path = f"/video/{youtube_id}"
+        ta_video = TubeArchivist().get(path)
+
+        return ta_video.get("channel")
+
+    def update_metadata(self, ta_channel):
+        """update channel metadata"""
+        path = "Items/" + self.show["Id"]
+        data = {
+            "Id": self.show["Id"],
+            "Name": ta_channel.get("channel_name"),
+            "Overview": ta_channel.get("channel_description"),
+            "Genres": [],
+            "Tags": [],
+            "ProviderIds": {},
+        }
+        Jellyfin().post(path, data)
+
+    def update_artwork(self, ta_channel):
+        """set channel artwork"""
+        jf_id = self.show["Id"]
+        jf_handler = Jellyfin()
+        primary = TubeArchivist().get_thumb(ta_channel["channel_thumb_url"])
+        jf_handler.post_img(f"Items/{jf_id}/Images/Primary", primary)
+        jf_handler.post_img(f"Items/{jf_id}/Images/Logo", primary)
+        banner = TubeArchivist().get_thumb(ta_channel["channel_banner_url"])
+        jf_handler.post_img(f"Items/{jf_id}/Images/Banner", banner)
+        tvart = TubeArchivist().get_thumb(ta_channel["channel_tvart_url"])
+        jf_handler.post_img(f"Items/{jf_id}/Images/Backdrop", tvart)
+
+    def validate_episodes(self):
+        """sync all episodes"""
+        all_episodes = self._get_all_episodes()
+        for video in all_episodes:
+            youtube_id = os.path.split(video["Path"])[-1][9:20]
+            jf_id = video["Id"]
+            Episode(youtube_id, jf_id).sync()