Add pytest, #build

Changed:
- Fixed running tests problem
- Added some simple unit tests
- Add to queue error handling improvement
- Fixed is_shorts from id
- Fixed logout

.github/workflows/lint_js.yml

@@ -1,16 +1,22 @@
 name: lint_js
 
-on: [pull_request, push]
+on:
+  push:
+    paths:
+      - '**/*.js'
+  pull_request:
+    paths:
+      - '**/*.js'
 
 jobs:
   check:
     name: lint_js
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-node@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
         with:
-          node-version: '16'
+          node-version: '22'
       - run: npm ci
       - run: npm run lint
       - run: npm run format -- --check

.github/workflows/lint_python.yml

@@ -1,14 +1,42 @@
 name: lint_python
-on: [pull_request, push]
+
+on:
+  push:
+    paths:
+      - '**/*.py'
+  pull_request:
+    paths:
+      - '**/*.py'
+
 jobs:
   lint_python:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y gcc libldap2-dev libsasl2-dev libssl-dev
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
-      - run: pip install --upgrade pip wheel
-      - run: pip install bandit black codespell flake8 flake8-bugbear
-                         flake8-comprehensions isort requests
-      - run: ./deploy.sh validate
+          python-version: '3.11'
+
+      - name: Cache pip
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
+      - name: Install python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r tubearchivist/requirements-dev.txt
+
+      - name: Run Linter
+        run: ./deploy.sh validate

.github/workflows/unit_tests.yml

@@ -0,0 +1,43 @@
+name: python_unit_tests
+
+on:
+  push:
+    paths:
+      - '**/*.py'
+  pull_request:
+    paths:
+      - '**/*.py'
+
+jobs:
+  unit-tests:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y gcc libldap2-dev libsasl2-dev libssl-dev
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Cache pip
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r tubearchivist/requirements-dev.txt
+
+      - name: Run unit tests
+        run: pytest tubearchivist

Dockerfile

@@ -44,7 +44,7 @@ RUN apt-get clean && apt-get -y update && apt-get -y install --no-install-recomm
 RUN if [ "$INSTALL_DEBUG" ] ; then \
         apt-get -y update && apt-get -y install --no-install-recommends \
         vim htop bmon net-tools iputils-ping procps \
-        && pip install --user ipython \
+        && pip install --user ipython pytest pytest-django \
     ; fi
 
 # make folders

deploy.sh

@@ -50,6 +50,7 @@ function sync_test {
         --exclude ".gitignore" \
         --exclude "**/cache" \
         --exclude "**/__pycache__/" \
+        --exclude "**/.pytest_cache/" \
         --exclude ".venv" \
         --exclude "db.sqlite3" \
         --exclude ".mypy_cache" \

tubearchivist/api/src/search_processor.py

@@ -7,7 +7,7 @@ Functionality:
 import urllib.parse
 
 from home.src.download.thumbnails import ThumbManager
-from home.src.ta.helper import date_praser, get_duration_str
+from home.src.ta.helper import date_parser, get_duration_str
 from home.src.ta.settings import EnvironmentSettings
 
 
@@ -67,7 +67,7 @@ class SearchProcess:
         """run on single channel"""
         channel_id = channel_dict["channel_id"]
         art_base = f"/cache/channels/{channel_id}"
-        date_str = date_praser(channel_dict["channel_last_refresh"])
+        date_str = date_parser(channel_dict["channel_last_refresh"])
         channel_dict.update(
             {
                 "channel_last_refresh": date_str,
@@ -83,8 +83,8 @@ class SearchProcess:
         """run on single video dict"""
         video_id = video_dict["youtube_id"]
         media_url = urllib.parse.quote(video_dict["media_url"])
-        vid_last_refresh = date_praser(video_dict["vid_last_refresh"])
-        published = date_praser(video_dict["published"])
+        vid_last_refresh = date_parser(video_dict["vid_last_refresh"])
+        published = date_parser(video_dict["published"])
         vid_thumb_url = ThumbManager(video_id).vid_thumb_path()
         channel = self._process_channel(video_dict["channel"])
 
@@ -109,7 +109,7 @@ class SearchProcess:
     def _process_playlist(playlist_dict):
         """run on single playlist dict"""
         playlist_id = playlist_dict["playlist_id"]
-        playlist_last_refresh = date_praser(
+        playlist_last_refresh = date_parser(
             playlist_dict["playlist_last_refresh"]
         )
         playlist_dict.update(
@@ -125,7 +125,7 @@ class SearchProcess:
         """run on single download item"""
         video_id = download_dict["youtube_id"]
         vid_thumb_url = ThumbManager(video_id).vid_thumb_path()
-        published = date_praser(download_dict["published"])
+        published = date_parser(download_dict["published"])
 
         download_dict.update(
             {

tubearchivist/api/views.py

@@ -688,14 +688,7 @@ class DownloadApiListView(ApiBaseView):
 
         pending = [i["youtube_id"] for i in to_add if i["status"] == "pending"]
         url_str = " ".join(pending)
-        try:
-            youtube_ids = Parser(url_str).parse()
-        except ValueError:
-            message = f"failed to parse: {url_str}"
-            print(message)
-            return Response({"message": message}, status=400)
-
-        extrac_dl.delay(youtube_ids, auto_start=auto_start)
+        extrac_dl.delay(url_str, auto_start=auto_start)
 
         return Response(data)
 

tubearchivist/config/settings.py

@@ -33,7 +33,9 @@ SECRET_KEY = PW_HASH.hexdigest()
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = bool(environ.get("DJANGO_DEBUG"))
 
-ALLOWED_HOSTS, CSRF_TRUSTED_ORIGINS = ta_host_parser(environ["TA_HOST"])
+ALLOWED_HOSTS, CSRF_TRUSTED_ORIGINS = ta_host_parser(
+    environ.get("TA_HOST", "localhost")
+)
 
 # Application definition
 
@@ -241,7 +243,11 @@ USE_TZ = True
 STATIC_URL = "/static/"
 STATICFILES_DIRS = (str(BASE_DIR.joinpath("static")),)
 STATIC_ROOT = str(BASE_DIR.joinpath("staticfiles"))
-STATICFILES_STORAGE = "whitenoise.storage.CompressedManifestStaticFilesStorage"
+STORAGES = {
+    "staticfiles": {
+        "BACKEND": "whitenoise.storage.CompressedManifestStaticFilesStorage",
+    },
+}
 
 # Default primary key field type
 # https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field

tubearchivist/home/celery.py

@@ -3,10 +3,8 @@
 import os
 
 from celery import Celery
-from home.src.ta.config import AppConfig
 from home.src.ta.settings import EnvironmentSettings
 
-CONFIG = AppConfig().config
 REDIS_HOST = EnvironmentSettings.REDIS_HOST
 REDIS_PORT = EnvironmentSettings.REDIS_PORT
 

tubearchivist/home/src/index/reindex.py

@@ -375,6 +375,7 @@ class Reindex(ReindexBase):
             channel.json_data["channel_overwrites"] = overwrites
 
         channel.upload_to_es()
+        channel.sync_to_videos()
         ChannelFullScan(channel_id).scan()
         self.processed["channels"] += 1
 

tubearchivist/home/src/ta/helper.py

@@ -93,12 +93,14 @@ def requests_headers() -> dict[str, str]:
     return {"User-Agent": template}
 
 
-def date_praser(timestamp: int | str) -> str:
+def date_parser(timestamp: int | str) -> str:
     """return formatted date string"""
     if isinstance(timestamp, int):
         date_obj = datetime.fromtimestamp(timestamp)
     elif isinstance(timestamp, str):
         date_obj = datetime.strptime(timestamp, "%Y-%m-%d")
+    else:
+        raise TypeError(f"invalid timestamp: {timestamp}")
 
     return date_obj.date().isoformat()
 
@@ -138,8 +140,9 @@ def get_mapping() -> dict:
 def is_shorts(youtube_id: str) -> bool:
     """check if youtube_id is a shorts video, bot not it it's not a shorts"""
     shorts_url = f"https://www.youtube.com/shorts/{youtube_id}"
+    cookies = {"SOCS": "CAI"}
     response = requests.head(
-        shorts_url, headers=requests_headers(), timeout=10
+        shorts_url, cookies=cookies, headers=requests_headers(), timeout=10
     )
 
     return response.status_code == 200
@@ -183,6 +186,8 @@ def get_duration_str(seconds: int) -> str:
             unit_count, seconds = divmod(seconds, unit_seconds)
             duration_parts.append(f"{unit_count:02}{unit_label}")
 
+    duration_parts[0] = duration_parts[0].lstrip("0")
+
     return " ".join(duration_parts)
 
 

tubearchivist/home/src/ta/urlparser.py

@@ -42,6 +42,10 @@ class Parser:
             youtube_id = parsed.path.strip("/")
             return self._validate_expected(youtube_id, "video")
 
+        if "youtube.com" not in parsed.netloc:
+            message = f"invalid domain: {parsed.netloc}"
+            raise ValueError(message)
+
         query_parsed = parse_qs(parsed.query)
         if "v" in query_parsed:
             # video from v query str

tubearchivist/home/templates/home/base.html

@@ -60,7 +60,10 @@
                     <a href="{% url 'settings' %}">
                         <img src="{% static 'img/icon-gear.svg' %}" alt="gear-icon" title="Settings">
                     </a>
-                    <a href="{% url 'logout' %}">
+                    <form id="logout-form" action="{% url 'logout' %}" method="post" style="display:none;">
+                        {% csrf_token %}
+                    </form>
+                    <a href="#" onclick="document.getElementById('logout-form').submit();">
                         <img class="alert-hover" src="{% static 'img/icon-exit.svg' %}" alt="exit-icon" title="Logout">
                     </a>
                 </div>

tubearchivist/home/tests/conftest.py

@@ -0,0 +1,11 @@
+"""test configs"""
+
+import os
+
+import pytest
+
+
+@pytest.fixture(scope="session", autouse=True)
+def change_test_dir(request):
+    """change directory to project folder"""
+    os.chdir(request.config.rootdir / "tubearchivist")

tubearchivist/home/tests/test_ta/test_helper.py

@@ -0,0 +1,113 @@
+"""tests for helper functions"""
+
+import pytest
+from home.src.ta.helper import (
+    date_parser,
+    get_duration_str,
+    get_mapping,
+    is_shorts,
+    randomizor,
+    time_parser,
+)
+
+
+def test_randomizor_with_positive_length():
+    """test randomizer"""
+    length = 10
+    result = randomizor(length)
+    assert len(result) == length
+    assert result.isalnum()
+
+
+def test_date_parser_with_int():
+    """unix timestamp"""
+    timestamp = 1621539600
+    expected_date = "2021-05-20"
+    assert date_parser(timestamp) == expected_date
+
+
+def test_date_parser_with_str():
+    """iso timestamp"""
+    date_str = "2021-05-21"
+    expected_date = "2021-05-21"
+    assert date_parser(date_str) == expected_date
+
+
+def test_date_parser_with_invalid_input():
+    """invalid type"""
+    invalid_input = [1621539600]
+    with pytest.raises(TypeError):
+        date_parser(invalid_input)
+
+
+def test_date_parser_with_invalid_string_format():
+    """invalid date string"""
+    invalid_date_str = "21/05/2021"
+    with pytest.raises(ValueError):
+        date_parser(invalid_date_str)
+
+
+def test_time_parser_with_numeric_string():
+    """as number"""
+    timestamp = "100"
+    expected_seconds = 100
+    assert time_parser(timestamp) == expected_seconds
+
+
+def test_time_parser_with_hh_mm_ss_format():
+    """to seconds"""
+    timestamp = "01:00:00"
+    expected_seconds = 3600.0
+    assert time_parser(timestamp) == expected_seconds
+
+
+def test_time_parser_with_empty_string():
+    """handle empty"""
+    timestamp = ""
+    assert time_parser(timestamp) is False
+
+
+def test_time_parser_with_invalid_format():
+    """not enough to unpack"""
+    timestamp = "01:00"
+    with pytest.raises(ValueError):
+        time_parser(timestamp)
+
+
+def test_time_parser_with_non_numeric_input():
+    """non numeric"""
+    timestamp = "1a:00:00"
+    with pytest.raises(ValueError):
+        time_parser(timestamp)
+
+
+def test_get_mapping():
+    """test mappint"""
+    index_config = get_mapping()
+    assert isinstance(index_config, list)
+    assert all(isinstance(i, dict) for i in index_config)
+
+
+def test_is_shorts():
+    """is shorts id"""
+    youtube_id = "YG3-Pw3rixU"
+    assert is_shorts(youtube_id)
+
+
+def test_is_not_shorts():
+    """is not shorts id"""
+    youtube_id = "Ogr9kbypSNg"
+    assert is_shorts(youtube_id) is False
+
+
+def test_get_duration_str():
+    """only seconds"""
+    assert get_duration_str(None) == "NA"
+    assert get_duration_str(5) == "5s"
+    assert get_duration_str(10) == "10s"
+    assert get_duration_str(500) == "8m 20s"
+    assert get_duration_str(1000) == "16m 40s"
+    assert get_duration_str(5000) == "1h 23m 20s"
+    assert get_duration_str(500000) == "5d 18h 53m 20s"
+    assert get_duration_str(5000000) == "57d 20h 53m 20s"
+    assert get_duration_str(50000000) == "1y 213d 16h 53m 20s"

tubearchivist/home/tests/test_ta/test_urlparser.py

@@ -0,0 +1,144 @@
+"""tests for url parser"""
+
+import pytest
+from home.src.ta.urlparser import Parser
+
+# video id parsing
+VIDEO_URL_IN = [
+    "7DKv5H5Frt0",
+    "https://www.youtube.com/watch?v=7DKv5H5Frt0",
+    "https://www.youtube.com/watch?v=7DKv5H5Frt0&t=113&feature=shared",
+    "https://www.youtube.com/watch?v=7DKv5H5Frt0&list=PL96C35uN7xGJu6skU4TBYrIWxggkZBrF5&index=1&pp=iAQB"  # noqa: E501
+    "https://youtu.be/7DKv5H5Frt0",
+]
+VIDEO_OUT = [{"type": "video", "url": "7DKv5H5Frt0", "vid_type": "unknown"}]
+VIDEO_TEST_CASES = [(i, VIDEO_OUT) for i in VIDEO_URL_IN]
+
+# shorts id parsing
+SHORTS_URL_IN = [
+    "https://www.youtube.com/shorts/YG3-Pw3rixU",
+    "https://youtube.com/shorts/YG3-Pw3rixU?feature=shared",
+]
+SHORTS_OUT = [{"type": "video", "url": "YG3-Pw3rixU", "vid_type": "shorts"}]
+SHORTS_TEST_CASES = [(i, SHORTS_OUT) for i in SHORTS_URL_IN]
+
+# channel id parsing
+CHANNEL_URL_IN = [
+    "UCBa659QWEk1AI4Tg--mrJ2A",
+    "@TomScottGo",
+    "https://www.youtube.com/channel/UCBa659QWEk1AI4Tg--mrJ2A",
+    "https://www.youtube.com/@TomScottGo",
+]
+CHANNEL_OUT = [
+    {
+        "type": "channel",
+        "url": "UCBa659QWEk1AI4Tg--mrJ2A",
+        "vid_type": "unknown",
+    }
+]
+CHANNEL_TEST_CASES = [(i, CHANNEL_OUT) for i in CHANNEL_URL_IN]
+
+# channel vid type parsing
+CHANNEL_VID_TYPES = [
+    (
+        "https://www.youtube.com/@IBRACORP/videos",
+        [
+            {
+                "type": "channel",
+                "url": "UC7aW7chIafJG6ECYAd3N5uQ",
+                "vid_type": "videos",
+            }
+        ],
+    ),
+    (
+        "https://www.youtube.com/@IBRACORP/shorts",
+        [
+            {
+                "type": "channel",
+                "url": "UC7aW7chIafJG6ECYAd3N5uQ",
+                "vid_type": "shorts",
+            }
+        ],
+    ),
+    (
+        "https://www.youtube.com/@IBRACORP/streams",
+        [
+            {
+                "type": "channel",
+                "url": "UC7aW7chIafJG6ECYAd3N5uQ",
+                "vid_type": "streams",
+            }
+        ],
+    ),
+]
+
+# playlist id parsing
+PLAYLIST_URL_IN = [
+    "PL96C35uN7xGJu6skU4TBYrIWxggkZBrF5",
+    "https://www.youtube.com/playlist?list=PL96C35uN7xGJu6skU4TBYrIWxggkZBrF5",
+]
+PLAYLIST_OUT = [
+    {
+        "type": "playlist",
+        "url": "PL96C35uN7xGJu6skU4TBYrIWxggkZBrF5",
+        "vid_type": "unknown",
+    }
+]
+PLAYLIST_TEST_CASES = [(i, PLAYLIST_OUT) for i in PLAYLIST_URL_IN]
+
+# personal playlists
+EXPECTED_WL = [{"type": "playlist", "url": "WL", "vid_type": "unknown"}]
+EXPECTED_LL = [{"type": "playlist", "url": "LL", "vid_type": "unknown"}]
+PERSONAL_PLAYLISTS_TEST_CASES = [
+    ("WL", EXPECTED_WL),
+    ("https://www.youtube.com/playlist?list=WL", EXPECTED_WL),
+    ("LL", EXPECTED_LL),
+    ("https://www.youtube.com/playlist?list=LL", EXPECTED_LL),
+]
+
+# collect tests expected to pass
+PASSTING_TESTS = []
+PASSTING_TESTS.extend(VIDEO_TEST_CASES)
+PASSTING_TESTS.extend(SHORTS_TEST_CASES)
+PASSTING_TESTS.extend(CHANNEL_TEST_CASES)
+PASSTING_TESTS.extend(CHANNEL_VID_TYPES)
+PASSTING_TESTS.extend(PLAYLIST_TEST_CASES)
+PASSTING_TESTS.extend(PERSONAL_PLAYLISTS_TEST_CASES)
+
+
+@pytest.mark.parametrize("url_str, expected_result", PASSTING_TESTS)
+def test_passing_parse(url_str, expected_result):
+    """test parser"""
+    parser = Parser(url_str)
+    parsed = parser.parse()
+    assert parsed == expected_result
+
+
+INVALID_IDS_ERRORS = [
+    "aaaaa",
+    "https://www.youtube.com/playlist?list=AAAA",
+    "https://www.youtube.com/channel/UC9-y-6csu5WGm29I7Jiwpn",
+    "https://www.youtube.com/watch?v=CK3_zarXkw",
+]
+
+
+@pytest.mark.parametrize("invalid_value", INVALID_IDS_ERRORS)
+def test_invalid_ids(invalid_value):
+    """test for invalid IDs"""
+    with pytest.raises(ValueError, match="not a valid id_str"):
+        parser = Parser(invalid_value)
+        parser.parse()
+
+
+INVALID_DOMAINS = [
+    "https://vimeo.com/32001208",
+    "https://peertube.tv/w/8RiJE2j2nw569FVgPNjDt7",
+]
+
+
+@pytest.mark.parametrize("invalid_value", INVALID_DOMAINS)
+def test_invalid_domains(invalid_value):
+    """raise error on none YT domains"""
+    parser = Parser(invalid_value)
+    with pytest.raises(ValueError, match="invalid domain"):
+        parser.parse()

tubearchivist/home/tests/test_views.py

@@ -1,42 +0,0 @@
-"""All test classes"""
-
-from django.test import TestCase
-
-
-class URLTests(TestCase):
-    """test if all expected URL are there"""
-
-    def test_home_view(self):
-        """check homepage"""
-        response = self.client.get("/")
-        self.assertEqual(response.status_code, 200)
-
-    def test_about_view(self):
-        """check about page"""
-        response = self.client.get("/about/")
-        self.assertEqual(response.status_code, 200)
-
-    def test_downloads_view(self):
-        """check downloads page"""
-        response = self.client.get("/downloads/")
-        self.assertEqual(response.status_code, 200)
-
-    def test_channel_view(self):
-        """check channel page"""
-        response = self.client.get("/channel/")
-        self.assertEqual(response.status_code, 200)
-
-    def test_settings_view(self):
-        """check settings page"""
-        response = self.client.get("/settings/")
-        self.assertEqual(response.status_code, 200)
-
-    def test_progress_view(self):
-        """check ajax progress endpoint"""
-        response = self.client.get("/downloads/progress/")
-        self.assertEqual(response.status_code, 200)
-
-    def test_process_view(self):
-        """check process ajax endpoint"""
-        response = self.client.get("/process/")
-        self.assertEqual(response.status_code, 200)

tubearchivist/requirements-dev.txt

@@ -5,4 +5,6 @@ flake8
 isort
 pylint
 pylint-django
+pytest
+pytest-django
 types-requests

tubearchivist/requirements.txt

@@ -7,8 +7,8 @@ django-cors-headers==4.3.1
 djangorestframework==3.15.1
 Pillow==10.3.0
 redis==5.0.4
-requests==2.31.0
+requests==2.32.1
 ryd-client==0.0.6
 uWSGI==2.0.25.1
 whitenoise==6.6.0
-yt-dlp @ git+https://github.com/bbilly1/yt-dlp@54b823be28f396608349cca69d52eb4c4b72b8b0
+yt-dlp @ git+https://github.com/yt-dlp/yt-dlp@8e15177b4113c355989881e4e030f695a9b59c3a