Compare commits
386 Commits
Author | SHA1 | Date |
---|---|---|
Simon | ee5d73917c | |
Simon | 478d05997c | |
Simon | 0e305f60f4 | |
Simon | 3c441c3a31 | |
Simon | 228677ddfb | |
Simon | dc0c82c814 | |
Simon | 34ad0ca07d | |
Simon | c82d9e2140 | |
Simon | bbf59eaca9 | |
Simon | e42737ad9b | |
Simon | 1d9c274390 | |
Simon | 2a35b42d88 | |
Simon | 11ab314649 | |
Simon | d58d133baf | |
Simon | d6f3fd883c | |
Simon | 8f1a5c8557 | |
Simon | fdc427977e | |
Simon | db080e97bb | |
Simon | 5235af3d91 | |
Simon | 6ab70c7602 | |
Simon | 86d157699a | |
Simon | c176405b32 | |
Simon | 6e7cb74366 | |
Simon | 05cfeb9d99 | |
Simon | 4edb5adead | |
Simon | c17627a911 | |
Simon | 36a738d5d7 | |
Simon | d6c4a6ea46 | |
Simon | 77ee9cfc13 | |
Simon | c413811e17 | |
Simon | 2a9769d154 | |
Simon | d9ce9641e2 | |
Simon | f874d402b1 | |
Simon | 97b6d7d606 | |
Simon | 4a38636ef3 | |
Simon | 97bc03f855 | |
Simon | 770990c568 | |
Simon | ddc4685811 | |
Simon | 56220a94e0 | |
Simon | fd039de53d | |
Simon | 320ead0bd2 | |
Simon | e33341d30d | |
Simon | 21b79e7c8f | |
Simon | 9366b8eab9 | |
Simon | 011073617d | |
Simon | 784f90b16d | |
Simon | c1cd9bc8eb | |
Simon | e0f1828d9c | |
Simon | f5a2e624d8 | |
Simon | dc08c83da5 | |
Simon | 33ecd73137 | |
Simon | cb6476fa8c | |
Simon | ec64a88d1e | |
Simon | 0c487e6339 | |
Simon | f7ad1000c7 | |
Simon | aecd189d04 | |
Simon | b735a770e3 | |
Simon | 5c84a2cbf8 | |
Simon | a4d062fa52 | |
Simon | 9c34bb01d9 | |
Simon | 8c38a2eb69 | |
Simon | 852abf254d | |
Simon | 25edff28e7 | |
lamusmaser | 731f4b6111 | |
Simon | e512329599 | |
Simon | e26b039899 | |
Simon | 8bf7f71351 | |
Simon | a72be27982 | |
Simon | b2c1b417e5 | |
Simon | a348b4a810 | |
Simon | bb8db53f7d | |
Simon | 2711537a4d | |
dot-mike | 45f455070d | |
Simon | 6dcef70b8e | |
Simon | c993a5de5c | |
Greg | 090d88c336 | |
Nick | 0e967d721f | |
Simon | c32dbf8bc8 | |
dot-mike | df08a6d591 | |
DarkFighterLuke | 9339b9227e | |
Simon | 8778546577 | |
Simon | 0ff27ebfb9 | |
Simon | 0d863ef557 | |
Simon | 56ca49d0e2 | |
Simon | 27b6efcab7 | |
Simon | 18ba808664 | |
Simon | 65738ef52c | |
Simon | 4049a2a3c1 | |
PhuriousGeorge | 49659322a1 | |
Simon | 4078eb307f | |
Daniel Jue | 7f056b38f4 | |
Simon | 86fe31d258 | |
Simon | 5b26433599 | |
Simon | 4d2fc5423e | |
Simon | 94295cdbd4 | |
Simon | b84bf78974 | |
Simon | 14e23a4371 | |
Simon | fe8f4faa10 | |
Simon | ddc0b7a481 | |
Simon | 7eec3ece49 | |
Simon | 789c35e2b5 | |
Simon | 8870782a6e | |
Simon | e75ffb603c | |
Simon | feabc87c9f | |
Simon | 6f1a45ffb1 | |
Simon | 098db97cba | |
Simon | 597da56975 | |
Simon | 325bdf5cba | |
Simon | db2f249979 | |
Simon | b61b8635b8 | |
Simon | 5aafc21bda | |
lamusmaser | 099c70a13b | |
Simon | 43708ee2a3 | |
Simon | cfb15c1a78 | |
Simon | e9a95d7ada | |
Simon | a21a111221 | |
Simon | 18e504faf2 | |
Simon | 9ffe2098a5 | |
Simon | 1315e836a4 | |
Simon | 2e4289e75c | |
Simon | 96e73a3a53 | |
Simon | a369be0f4a | |
Simon | d5676e5173 | |
Simon | 44c4cf93e2 | |
Simon | 02ac590caa | |
Simon | a466c02304 | |
Simon | e74c26fe36 | |
Simon | b1267cba83 | |
Simon | 91bb0ed9c0 | |
Simon | 4a145ee7cb | |
Simon | 463019ce5a | |
Simon | 9a9d35cac4 | |
Simon | f41ecd24c5 | |
crocs | eced8200c1 | |
Simon | 669bc6a620 | |
lamusmaser | 37df9b65c7 | |
lamusmaser | 6721d01fa6 | |
crocs | 2b49af9620 | |
Derek Slenk | 2f62898a10 | |
spechter | 832259ce48 | |
Simon | b8ccce250a | |
Simon | aa04ecff4f | |
Simon | dcf97d3d24 | |
crocs | 879ad52b32 | |
Simon | 0bedc3ee93 | |
Simon | 1657c55cbe | |
Simon | 8b1324139d | |
Simon | 04124e3dad | |
Simon | 9c26357f76 | |
extome | 7133d6b441 | |
Simon | 6bc0111d0a | |
Simon | 1188e66f37 | |
Simon | ef6d3e868d | |
Simon | d677f9579e | |
Simon | 0b920e87ae | |
Simon | 4d5aa4ad2f | |
Simon | 4b63c2f536 | |
Simon | 31ad9424f5 | |
Simon | 45f4ccfd93 | |
Simon | 285e2042ae | |
Simon | e4b7f8ce38 | |
Simon | 6892cbbc19 | |
Simon | 58ea256b44 | |
Merlin | aa475c58aa | |
Simon | 8247314d01 | |
Simon | 2826ca4a43 | |
Simon | 64ffc18da7 | |
Simon | 21fde5e068 | |
Simon | ea9ed6c238 | |
Simon | 8eaed07cff | |
Clark | 4d111aff82 | |
Simon | 7236bea29a | |
Simon | 5165c3e34a | |
Simon | 572b23169c | |
Steve Ovens | e1fce06f97 | |
Simon | 446d5b7949 | |
Simon | 17c0310220 | |
Omar Laham | 1b0be84972 | |
Simon | 2df68fa83c | |
Simon | 4184736bee | |
Simon | 81a5f15600 | |
Simon | 4a4a274259 | |
Simon | 0776cea7bc | |
Simon | fb853e6c73 | |
Simon | 57d8b060d2 | |
Simon | 6d1810946b | |
Simon | 88f230c3f4 | |
Simon | e9eddf06fb | |
Simon | 8af7a3caf4 | |
Simon | ad7f1f05b0 | |
Simon | e1fe8d1e29 | |
Simon | f8f01ac27f | |
Simon | 8e79cba7d5 | |
Simon | 87e457401d | |
Simon | bb271e276c | |
Simon | 9967015eba | |
Simon | 3b7e4c9266 | |
Xavier Chevalier | 1dd3fb9341 | |
Simon | 120f9e468d | |
Simon | 88f5c58b8e | |
Simon | 6bd06f61cf | |
Igor Rzegocki | 6a83756fb4 | |
Simon | 515b724047 | |
Simon | 77fef5de57 | |
Simon | 9d09d27fba | |
Simon | 0e767e2f84 | |
Simon | 7801ed0d60 | |
Igor Rzegocki | 6abec9401b | |
Simon | 1cdb9e1ad5 | |
Simon | 7afeb41469 | |
Simon | bae11fe1f1 | |
Simon | 0cacaee213 | |
Simon | dcbd8d2a55 | |
Simon | 892e81c185 | |
Igor Rzegocki | f423ddc53a | |
Igor Rzegocki | b2bb7ea28e | |
Simon | 38b3815a33 | |
Simon | 92975a5c95 | |
Joseph Liu | a5b61bfaf6 | |
Clark | 85b56300b3 | |
Kevin Gibbons | 8fa9e23c6e | |
Simon | a7fc7902f0 | |
Simon | 879f5ab52f | |
Simon | c6458c6ec1 | |
Simon | 47c433e7c1 | |
Simon | dc41e5062d | |
Merlin | 317942b7e1 | |
Merlin | 65d768bf02 | |
Merlin | 0767bbfecf | |
Simon | 78d6699c68 | |
Simon | a807d53ff8 | |
Simon | fa45cf08ba | |
Simon | c3da3e23af | |
Simon | 5cf5b4edb7 | |
Simon | 0c9c88fc0b | |
Simon | 725bba0963 | |
Simon | 76981635dc | |
Simon | b56316932f | |
Simon | 8dba2e240e | |
Simon | 4016e81f9a | |
Simon | 5ee37eb0cb | |
Simon | 4650963cc7 | |
Simon | 5acc1ea718 | |
Simon | 505f5b73c5 | |
Simon | d491b0b347 | |
Simon | 52d6c59f3f | |
Simon | 4afb605736 | |
Clark | fcc1c2a648 | |
Simon | 4ded8988c3 | |
Simon | 988c2b8af7 | |
Simon | 58ef8f753f | |
Simon | 3e9f1a392a | |
Simon | 2563722f16 | |
Simon | fb089dd3de | |
dmynerd78 | 983612f460 | |
Simon | d42bd612d0 | |
Simon | 41f6a03751 | |
Simon | f1e25c9a20 | |
Simon | 15794ebfc8 | |
Simon | 68928f5cb1 | |
Simon | a514dda1ff | |
Simon | 2bccb698e6 | |
Simon | 076452c612 | |
Simon | b005b7bcfe | |
Simon | a2eb42ebb9 | |
Simon | 33ff586af4 | |
Simon | 3803537739 | |
Simon | 6151da821f | |
Simon | 8f7f5e0e32 | |
Simon | fa140a3047 | |
Simon | 419b6f02a5 | |
Simon | 58818bb91c | |
Simon | b6ae225342 | |
Simon | 8411889db7 | |
Simon | 313bbe8b49 | |
Simon | 691c643745 | |
Simon | 9e8e929bcc | |
Simon | 2238565a94 | |
Simon | 39e9704436 | |
Simon | fa43753614 | |
Simon | 02be39b6ed | |
Simon | 375e1a2100 | |
Simon | e893dc3b24 | |
Simon | c1ea77434e | |
crocs | 0e1e544fee | |
Simon | a13cd2f7ba | |
Simon | befdc64772 | |
Simon | 06f3055913 | |
Simon | ca2c5b8dfc | |
Simon | c395a949cc | |
Simon | 4473e9c5b2 | |
Simon | 75a63c4828 | |
Simon | aea403a874 | |
Simon | ab8fed14bb | |
Simon | 6f915a5733 | |
Simon | f970ec867e | |
Simon | ef0d490890 | |
lamusmaser | 865089d46d | |
Simon | cd25eadd1c | |
Simon | d500fa5eeb | |
Simon | 4c681d813d | |
Simon | ddfab4a341 | |
Simon | 434aa97a86 | |
Simon | efde4b1142 | |
Simon | 6022bdd3cd | |
Simon | 99baf64b11 | |
Simon | 61b04ba5cf | |
Simon | 2a60360f4a | |
Simon | 8a7cb8bc6f | |
lamusmaser | 1be80b24c2 | |
Simon | 061c653bce | |
Simon | 72a98b0473 | |
Simon | 88e199ef9c | |
Simon | 879497d25a | |
Simon | 3f1d8cf75d | |
Simon | 32721cf7ed | |
Simon | 103409770d | |
Simon | 094ccf4186 | |
Simon | 247808563a | |
simon | 5927ced485 | |
simon | 6fb788b259 | |
simon | 5e92d06f21 | |
simon | 7082718c14 | |
simon | 7e2cd6acd3 | |
simon | 904d0de6aa | |
simon | 868247e6d4 | |
simon | c4e2332b83 | |
simon | 139d20560f | |
simon | 66a14cf389 | |
simon | 9b30c7df6e | |
simon | 5334d79d0d | |
simon | 64984bc1b3 | |
simon | 8ef59f5bff | |
simon | 9d6ab6b7b3 | |
simon | d62b0d3f8d | |
simon | 918a04c502 | |
simon | 60f1809ed8 | |
simon | f848e73251 | |
simon | c65fbb0b60 | |
simon | 95f114d817 | |
simon | 05eac1a8ca | |
simon | ea42f0f1e3 | |
simon | 625dc357cc | |
simon | e94e11c456 | |
simon | a9b5713629 | |
simon | dbaa13bfb0 | |
simon | 5d0d050149 | |
simon | c327e94726 | |
simon | 774780d520 | |
simon | 5e1167743f | |
simon | 4376b826c4 | |
simon | 0fef751ab5 | |
simon | 206921baf0 | |
simon | 0d2d3353a9 | |
simon | b47687535a | |
simon | e092a29b13 | |
simon | 170839362e | |
simon | b95a659396 | |
simon | 2b66786728 | |
simon | b7bfeaf215 | |
simon | cf37800c2b | |
simon | 5cc642098d | |
simon | 7c01ad88b2 | |
simon | e866bb3be5 | |
simon | 63021bd313 | |
simon | cbcb7484a7 | |
Dominik Sander | 1c0b407f3f | |
simon | 280c773441 | |
simon | efca460e9d | |
simon | 8f3b832069 | |
simon | 9b3d1fa1fd | |
Matthew Glinski | 9a38aff03d | |
simon | 06bbe2e400 | |
simon | 77900f89e3 | |
simon | bc39561606 | |
simon | 76535c6304 | |
simon | 790ba3d20e | |
simon | 89779ec13b | |
simon | 1b6b219e02 | |
simon | 5cd845e55d | |
simon | 3a091ac287 | |
simon | e385331f6c | |
simon | 4067b6c182 | |
simon | 3063236634 | |
simon | a17f05ef21 | |
simon | a4d42573ef |
|
@ -17,8 +17,5 @@ venv/
|
||||||
# Unneeded graphics
|
# Unneeded graphics
|
||||||
assets/*
|
assets/*
|
||||||
|
|
||||||
# Unneeded docs
|
|
||||||
docs/*
|
|
||||||
|
|
||||||
# for local testing only
|
# for local testing only
|
||||||
testing.sh
|
testing.sh
|
|
@ -38,6 +38,6 @@ body:
|
||||||
attributes:
|
attributes:
|
||||||
label: Relevant log output
|
label: Relevant log output
|
||||||
description: Please copy and paste any relevant Docker logs. This will be automatically formatted into code, so no need for backticks.
|
description: Please copy and paste any relevant Docker logs. This will be automatically formatted into code, so no need for backticks.
|
||||||
render: shell
|
render: Shell
|
||||||
validations:
|
validations:
|
||||||
required: true
|
required: true
|
||||||
|
|
|
@ -8,20 +8,7 @@ jobs:
|
||||||
- uses: actions/setup-python@v4
|
- uses: actions/setup-python@v4
|
||||||
with:
|
with:
|
||||||
python-version: '3.10'
|
python-version: '3.10'
|
||||||
# note: this logic is duplicated in the `validate` function in ./deploy.sh
|
|
||||||
# if you update this file, you should update that as well
|
|
||||||
- run: pip install --upgrade pip wheel
|
- run: pip install --upgrade pip wheel
|
||||||
- run: pip install bandit black codespell flake8 flake8-bugbear
|
- run: pip install bandit black codespell flake8 flake8-bugbear
|
||||||
flake8-comprehensions isort requests
|
flake8-comprehensions isort requests
|
||||||
- run: ./deploy.sh validate
|
- run: ./deploy.sh validate
|
||||||
# - run: black --check --diff --line-length 79 .
|
|
||||||
# - run: codespell --skip="./.git,./package.json,./package-lock.json,./node_modules"
|
|
||||||
# - run: flake8 . --count --max-complexity=10 --max-line-length=79
|
|
||||||
# --show-source --statistics
|
|
||||||
# - run: isort --check-only --line-length 79 --profile black .
|
|
||||||
# - run: pip install -r tubearchivist/requirements.txt
|
|
||||||
# - run: mkdir --parents --verbose .mypy_cache
|
|
||||||
# - run: mypy --ignore-missing-imports --install-types --non-interactive .
|
|
||||||
# - run: python3 tubearchivist/manage.py test || true
|
|
||||||
# - run: shopt -s globstar && pyupgrade --py36-plus **/*.py || true
|
|
||||||
# - run: safety check
|
|
||||||
|
|
|
@ -1,5 +1,6 @@
|
||||||
# python testing cache
|
# python testing cache
|
||||||
__pycache__
|
__pycache__
|
||||||
|
.venv
|
||||||
|
|
||||||
# django testing db
|
# django testing db
|
||||||
db.sqlite3
|
db.sqlite3
|
||||||
|
|
|
@ -1,19 +1,46 @@
|
||||||
## Contributing to Tube Archivist
|
# Contributing to Tube Archivist
|
||||||
|
|
||||||
Welcome, and thanks for showing interest in improving Tube Archivist!
|
Welcome, and thanks for showing interest in improving Tube Archivist!
|
||||||
|
|
||||||
## Table of Content
|
## Table of Content
|
||||||
|
- [Next Steps](#next-steps)
|
||||||
|
- [Beta Testing](#beta-testing)
|
||||||
- [How to open an issue](#how-to-open-an-issue)
|
- [How to open an issue](#how-to-open-an-issue)
|
||||||
- [Bug Report](#bug-report)
|
- [Bug Report](#bug-report)
|
||||||
- [Feature Request](#feature-request)
|
- [Feature Request](#feature-request)
|
||||||
- [Installation Help](#installation-help)
|
- [Installation Help](#installation-help)
|
||||||
- [How to make a Pull Request](#how-to-make-a-pull-request)
|
- [How to make a Pull Request](#how-to-make-a-pull-request)
|
||||||
|
- [Contributions beyond the scope](#contributions-beyond-the-scope)
|
||||||
|
- [User Scripts](#user-scripts)
|
||||||
- [Improve to the Documentation](#improve-to-the-documentation)
|
- [Improve to the Documentation](#improve-to-the-documentation)
|
||||||
- [Development Environment](#development-environment)
|
- [Development Environment](#development-environment)
|
||||||
---
|
---
|
||||||
|
|
||||||
|
## Next Steps
|
||||||
|
Going forward, this project will focus on developing a new modern frontend.
|
||||||
|
|
||||||
|
- For the time being, don't open any new PRs that are not towards the new frontend.
|
||||||
|
- New features requests likely won't get accepted during this process.
|
||||||
|
- Depending on the severity, bug reports may or may not get fixed during this time.
|
||||||
|
- When in doubt, reach out.
|
||||||
|
|
||||||
|
Join us on [Discord](https://tubearchivist.com/discord) if you want to help with that process.
|
||||||
|
|
||||||
|
## Beta Testing
|
||||||
|
Be the first to help test new features and improvements and provide feedback! There are regular `:unstable` builds for easy access. That's for the tinkerers and the breave. Ideally use a testing environment first, before a release be the first to install it on your main system.
|
||||||
|
|
||||||
|
There is always something that can get missed during development. Look at the commit messages tagged with `#build`, these are the unstable builds and give a quick overview what has changed.
|
||||||
|
|
||||||
|
- Test the features mentioned, play around, try to break it.
|
||||||
|
- Test the update path by installing the `:latest` release first, the upgrade to `:unstable` to check for any errors.
|
||||||
|
- Test the unstable build on a fresh install.
|
||||||
|
|
||||||
|
Then provide feedback, if there is a problem but also if there is no problem. Reach out on [Discord](https://tubearchivist.com/discord) in the `#beta-testing` channel with your findings.
|
||||||
|
|
||||||
|
This will help with a smooth update for the regular release. Plus you get to test things out early!
|
||||||
|
|
||||||
## How to open an issue
|
## How to open an issue
|
||||||
Please read this carefully before opening any [issue](https://github.com/tubearchivist/tubearchivist/issues) on GitHub.
|
Please read this carefully before opening any [issue](https://github.com/tubearchivist/tubearchivist/issues) on GitHub. Make sure you read [Next Steps](#next-steps) above.
|
||||||
|
|
||||||
**Do**:
|
**Do**:
|
||||||
- Do provide details and context, this matters a lot and makes it easier for people to help.
|
- Do provide details and context, this matters a lot and makes it easier for people to help.
|
||||||
|
@ -35,12 +62,12 @@ Please keep in mind:
|
||||||
- A bug that can't be reproduced, is difficult or sometimes even impossible to fix. Provide very clear steps *how to reproduce*.
|
- A bug that can't be reproduced, is difficult or sometimes even impossible to fix. Provide very clear steps *how to reproduce*.
|
||||||
|
|
||||||
### Feature Request
|
### Feature Request
|
||||||
This project needs your help to grow further. There is no shortage of ideas, see the open [issues on GH](https://github.com/tubearchivist/tubearchivist/issues?q=is%3Aopen+is%3Aissue+label%3Aenhancement) and the [roadmap](https://github.com/tubearchivist/tubearchivist#roadmap), what this project lacks is contributors to implement these ideas.
|
This project needs your help to grow further. There is no shortage of ideas, see the open [issues on GH](https://github.com/tubearchivist/tubearchivist/issues?q=is%3Aopen+is%3Aissue+label%3Aenhancement) and the [roadmap](https://github.com/tubearchivist/tubearchivist#roadmap), what this project lacks is contributors interested in helping with overall improvements of the application. Focus is *not* on adding new features, but improving existing ones.
|
||||||
|
|
||||||
Existing ideas are easily *multiple years* worth of development effort, at least at current speed. Best and fastest way to implement your feature is to do it yourself, that's why this project is open source after all. This project is *very* selective with accepting new feature requests at this point.
|
Existing ideas are easily *multiple years* worth of development effort, at least at current speed. This project is *very* selective with accepting new feature requests at this point.
|
||||||
|
|
||||||
Good feature requests usually fall into one or more of these categories:
|
Good feature requests usually fall into one or more of these categories:
|
||||||
- You want to work on your own idea within the next few days or weeks.
|
- You want to work on your own small scoped idea within the next few days or weeks.
|
||||||
- Your idea is beneficial for a wide range of users, not just for you.
|
- Your idea is beneficial for a wide range of users, not just for you.
|
||||||
- Your idea extends the current project by building on and improving existing functionality.
|
- Your idea extends the current project by building on and improving existing functionality.
|
||||||
- Your idea is quick and easy to implement, for an experienced as well as for a first time contributor.
|
- Your idea is quick and easy to implement, for an experienced as well as for a first time contributor.
|
||||||
|
@ -64,7 +91,11 @@ IMPORTANT: When receiving help, contribute back to the community by improving th
|
||||||
|
|
||||||
## How to make a Pull Request
|
## How to make a Pull Request
|
||||||
|
|
||||||
Thank you for contributing and helping improve this project. This is a quick checklist to help streamline the process:
|
Make sure you read [Next Steps](#next-steps) above.
|
||||||
|
|
||||||
|
Thank you for contributing and helping improve this project. Focus for the foreseeable future is on improving and building on existing functionality, *not* on adding and expanding the application.
|
||||||
|
|
||||||
|
This is a quick checklist to help streamline the process:
|
||||||
|
|
||||||
- For **code changes**, make your PR against the [testing branch](https://github.com/tubearchivist/tubearchivist/tree/testing). That's where all active development happens. This simplifies the later merging into *master*, minimizes any conflicts and usually allows for easy and convenient *fast-forward* merging.
|
- For **code changes**, make your PR against the [testing branch](https://github.com/tubearchivist/tubearchivist/tree/testing). That's where all active development happens. This simplifies the later merging into *master*, minimizes any conflicts and usually allows for easy and convenient *fast-forward* merging.
|
||||||
- For **documentation changes**, make your PR directly against the *master* branch.
|
- For **documentation changes**, make your PR directly against the *master* branch.
|
||||||
|
@ -87,6 +118,33 @@ to validate your changes. If you omit the path, all the project files will get c
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
## Contributions beyond the scope
|
||||||
|
|
||||||
|
As you have read the [FAQ](https://docs.tubearchivist.com/faq/) and the [known limitations](https://github.com/tubearchivist/tubearchivist#known-limitations) and have gotten an idea what this project tries to do, there will be some obvious shortcomings that stand out, that have been explicitly excluded from the scope of this project, at least for the time being.
|
||||||
|
|
||||||
|
Extending the scope of this project will only be feasible with more [regular contributors](https://github.com/tubearchivist/tubearchivist/graphs/contributors) that are willing to help improve this project in the long run. Contributors that have an overall improvement of the project in mind and not just about implementing this *one* thing.
|
||||||
|
|
||||||
|
Small minor additions, or making a PR for a documented feature request or bug, even if that was and will be your only contribution to this project, are always welcome and is *not* what this is about.
|
||||||
|
|
||||||
|
Beyond that, general rules to consider:
|
||||||
|
|
||||||
|
- Maintainability is key: It's not just about implementing something and being done with it, it's about maintaining it, fixing bugs as they occur, improving on it and supporting it in the long run.
|
||||||
|
- Others can do it better: Some problems have been solved by very talented developers. These things don't need to be reinvented again here in this project.
|
||||||
|
- Develop for the 80%: New features and additions *should* be beneficial for 80% of the users. If you are trying to solve your own problem that only applies to you, maybe that would be better to do in your own fork or if possible by a standalone implementation using the API.
|
||||||
|
- If all of that sounds too strict for you, as stated above, start becoming a regular contributor to this project.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## User Scripts
|
||||||
|
Some of you might have created useful scripts or API integrations around this project. Sharing is caring! Please add a link to your script to the Readme [here](https://github.com/tubearchivist/tubearchivist#user-scripts).
|
||||||
|
- Your repo should have a `LICENSE` file with one of the common open source licenses. People are expected to fork, adapt and build upon your great work.
|
||||||
|
- Your script should not modify the *official* files of Tube Archivist. E.g. your symlink script should build links *outside* of your `/youtube` folder. Or your fancy script that creates a beautiful artwork gallery should do that *outside* of the `/cache` folder. Modifying the *official* files and folders of TA are probably not supported.
|
||||||
|
- On the top of the repo you should have a mention and a link back to the Tube Archivist repo. Clearly state to **not** to open any issues on the main TA repo regarding your script.
|
||||||
|
- Example template:
|
||||||
|
- `[<user>/<repo>](https://linktoyourrepo.com)`: A short one line description.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
## Improve to the Documentation
|
## Improve to the Documentation
|
||||||
|
|
||||||
The documentation available at [docs.tubearchivist.com](https://docs.tubearchivist.com/) and is build from a separate repo [tubearchivist/docs](https://github.com/tubearchivist/docs). The Readme has additional instructions on how to make changes.
|
The documentation available at [docs.tubearchivist.com](https://docs.tubearchivist.com/) and is build from a separate repo [tubearchivist/docs](https://github.com/tubearchivist/docs). The Readme has additional instructions on how to make changes.
|
||||||
|
@ -136,14 +194,15 @@ bin/elasticsearch-service-tokens create elastic/kibana kibana
|
||||||
|
|
||||||
Example docker compose, use same version as for Elasticsearch:
|
Example docker compose, use same version as for Elasticsearch:
|
||||||
```yml
|
```yml
|
||||||
kibana:
|
services:
|
||||||
image: docker.elastic.co/kibana/kibana:0.0.0
|
kibana:
|
||||||
container_name: kibana
|
image: docker.elastic.co/kibana/kibana:0.0.0
|
||||||
environment:
|
container_name: kibana
|
||||||
- "ELASTICSEARCH_HOSTS=http://archivist-es:9200"
|
environment:
|
||||||
- "ELASTICSEARCH_SERVICEACCOUNTTOKEN=<your-token-here>"
|
- "ELASTICSEARCH_HOSTS=http://archivist-es:9200"
|
||||||
ports:
|
- "ELASTICSEARCH_SERVICEACCOUNTTOKEN=<your-token-here>"
|
||||||
- "5601:5601"
|
ports:
|
||||||
|
- "5601:5601"
|
||||||
```
|
```
|
||||||
|
|
||||||
If you want to run queries on the Elasticsearch container directly from your host with for example `curl` or something like *postman*, you might want to **publish** the port 9200 instead of just **exposing** it.
|
If you want to run queries on the Elasticsearch container directly from your host with for example `curl` or something like *postman*, you might want to **publish** the port 9200 instead of just **exposing** it.
|
||||||
|
|
38
Dockerfile
|
@ -1,20 +1,25 @@
|
||||||
# multi stage to build tube archivist
|
# multi stage to build tube archivist
|
||||||
# first stage to build python wheel, copy into final image
|
# build python wheel, download and extract ffmpeg, copy into final image
|
||||||
|
|
||||||
|
|
||||||
# First stage to build python wheel
|
# First stage to build python wheel
|
||||||
FROM python:3.10.9-slim-bullseye AS builder
|
FROM python:3.11.8-slim-bookworm AS builder
|
||||||
ARG TARGETPLATFORM
|
ARG TARGETPLATFORM
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
build-essential gcc libldap2-dev libsasl2-dev libssl-dev
|
build-essential gcc libldap2-dev libsasl2-dev libssl-dev git
|
||||||
|
|
||||||
# install requirements
|
# install requirements
|
||||||
COPY ./tubearchivist/requirements.txt /requirements.txt
|
COPY ./tubearchivist/requirements.txt /requirements.txt
|
||||||
RUN pip install --user -r requirements.txt
|
RUN pip install --user -r requirements.txt
|
||||||
|
|
||||||
|
# build ffmpeg
|
||||||
|
FROM python:3.11.8-slim-bookworm as ffmpeg-builder
|
||||||
|
COPY docker_assets/ffmpeg_download.py ffmpeg_download.py
|
||||||
|
RUN python ffmpeg_download.py $TARGETPLATFORM
|
||||||
|
|
||||||
# build final image
|
# build final image
|
||||||
FROM python:3.10.9-slim-bullseye as tubearchivist
|
FROM python:3.11.8-slim-bookworm as tubearchivist
|
||||||
|
|
||||||
ARG TARGETPLATFORM
|
ARG TARGETPLATFORM
|
||||||
ARG INSTALL_DEBUG
|
ARG INSTALL_DEBUG
|
||||||
|
@ -25,30 +30,15 @@ ENV PYTHONUNBUFFERED 1
|
||||||
COPY --from=builder /root/.local /root/.local
|
COPY --from=builder /root/.local /root/.local
|
||||||
ENV PATH=/root/.local/bin:$PATH
|
ENV PATH=/root/.local/bin:$PATH
|
||||||
|
|
||||||
|
# copy ffmpeg
|
||||||
|
COPY --from=ffmpeg-builder ./ffmpeg/ffmpeg /usr/bin/ffmpeg
|
||||||
|
COPY --from=ffmpeg-builder ./ffprobe/ffprobe /usr/bin/ffprobe
|
||||||
|
|
||||||
# install distro packages needed
|
# install distro packages needed
|
||||||
RUN apt-get clean && apt-get -y update && apt-get -y install --no-install-recommends \
|
RUN apt-get clean && apt-get -y update && apt-get -y install --no-install-recommends \
|
||||||
nginx \
|
nginx \
|
||||||
atomicparsley \
|
atomicparsley \
|
||||||
curl \
|
curl && rm -rf /var/lib/apt/lists/*
|
||||||
xz-utils && rm -rf /var/lib/apt/lists/*
|
|
||||||
|
|
||||||
# install patched ffmpeg build, default to linux64
|
|
||||||
RUN if [ "$TARGETPLATFORM" = "linux/arm64" ] ; then \
|
|
||||||
curl -s https://api.github.com/repos/yt-dlp/FFmpeg-Builds/releases/latest \
|
|
||||||
| grep browser_download_url \
|
|
||||||
| grep ".*master.*linuxarm64.*tar.xz" \
|
|
||||||
| cut -d '"' -f 4 \
|
|
||||||
| xargs curl -L --output ffmpeg.tar.xz ; \
|
|
||||||
else \
|
|
||||||
curl -s https://api.github.com/repos/yt-dlp/FFmpeg-Builds/releases/latest \
|
|
||||||
| grep browser_download_url \
|
|
||||||
| grep ".*master.*linux64.*tar.xz" \
|
|
||||||
| cut -d '"' -f 4 \
|
|
||||||
| xargs curl -L --output ffmpeg.tar.xz ; \
|
|
||||||
fi && \
|
|
||||||
tar -xf ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/bin/ "ffmpeg" && \
|
|
||||||
tar -xf ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/bin/ "ffprobe" && \
|
|
||||||
rm ffmpeg.tar.xz
|
|
||||||
|
|
||||||
# install debug tools for testing environment
|
# install debug tools for testing environment
|
||||||
RUN if [ "$INSTALL_DEBUG" ] ; then \
|
RUN if [ "$INSTALL_DEBUG" ] ; then \
|
||||||
|
|
54
README.md
|
@ -1,6 +1,6 @@
|
||||||
![Tube Archivist](assets/tube-archivist-banner.jpg?raw=true "Tube Archivist Banner")
|
![Tube Archivist](assets/tube-archivist-front.jpg?raw=true "Tube Archivist Banner")
|
||||||
|
[*more screenshots and video*](SHOWCASE.MD)
|
||||||
|
|
||||||
<h1 align="center">Your self hosted YouTube media server</h1>
|
|
||||||
<div align="center">
|
<div align="center">
|
||||||
<a href="https://github.com/bbilly1/tilefy" target="_blank"><img src="https://tiles.tilefy.me/t/tubearchivist-docker.png" alt="tubearchivist-docker" title="Tube Archivist Docker Pulls" height="50" width="190"/></a>
|
<a href="https://github.com/bbilly1/tilefy" target="_blank"><img src="https://tiles.tilefy.me/t/tubearchivist-docker.png" alt="tubearchivist-docker" title="Tube Archivist Docker Pulls" height="50" width="190"/></a>
|
||||||
<a href="https://github.com/bbilly1/tilefy" target="_blank"><img src="https://tiles.tilefy.me/t/tubearchivist-github-star.png" alt="tubearchivist-github-star" title="Tube Archivist GitHub Stars" height="50" width="190"/></a>
|
<a href="https://github.com/bbilly1/tilefy" target="_blank"><img src="https://tiles.tilefy.me/t/tubearchivist-github-star.png" alt="tubearchivist-github-star" title="Tube Archivist GitHub Stars" height="50" width="190"/></a>
|
||||||
|
@ -8,8 +8,6 @@
|
||||||
<a href="https://www.tubearchivist.com/discord" target="_blank"><img src="https://tiles.tilefy.me/t/tubearchivist-discord.png" alt="tubearchivist-discord" title="TA Discord Server Members" height="50" width="190"/></a>
|
<a href="https://www.tubearchivist.com/discord" target="_blank"><img src="https://tiles.tilefy.me/t/tubearchivist-discord.png" alt="tubearchivist-discord" title="TA Discord Server Members" height="50" width="190"/></a>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
![home screenshot](assets/tube-archivist-screenshot-home.png?raw=true "Tube Archivist Home")
|
|
||||||
|
|
||||||
## Table of contents:
|
## Table of contents:
|
||||||
* [Docs](https://docs.tubearchivist.com/) with [FAQ](https://docs.tubearchivist.com/faq/), and API documentation
|
* [Docs](https://docs.tubearchivist.com/) with [FAQ](https://docs.tubearchivist.com/faq/), and API documentation
|
||||||
* [Core functionality](#core-functionality)
|
* [Core functionality](#core-functionality)
|
||||||
|
@ -25,7 +23,7 @@
|
||||||
------------------------
|
------------------------
|
||||||
|
|
||||||
## Core functionality
|
## Core functionality
|
||||||
Once your YouTube video collection grows, it becomes hard to search and find a specific video. That's where Tube Archivist comes in: By indexing your video collection with metadata from YouTube, you can organize, search and enjoy your archived YouTube videos without hassle offline through a convenient web interface. THis includes:
|
Once your YouTube video collection grows, it becomes hard to search and find a specific video. That's where Tube Archivist comes in: By indexing your video collection with metadata from YouTube, you can organize, search and enjoy your archived YouTube videos without hassle offline through a convenient web interface. This includes:
|
||||||
* Subscribe to your favorite YouTube channels
|
* Subscribe to your favorite YouTube channels
|
||||||
* Download Videos using **yt-dlp**
|
* Download Videos using **yt-dlp**
|
||||||
* Index and make videos searchable
|
* Index and make videos searchable
|
||||||
|
@ -36,8 +34,8 @@ Once your YouTube video collection grows, it becomes hard to search and find a s
|
||||||
- [Discord](https://www.tubearchivist.com/discord): Connect with us on our Discord server.
|
- [Discord](https://www.tubearchivist.com/discord): Connect with us on our Discord server.
|
||||||
- [r/TubeArchivist](https://www.reddit.com/r/TubeArchivist/): Join our Subreddit.
|
- [r/TubeArchivist](https://www.reddit.com/r/TubeArchivist/): Join our Subreddit.
|
||||||
- [Browser Extension](https://github.com/tubearchivist/browser-extension) Tube Archivist Companion, for [Firefox](https://addons.mozilla.org/addon/tubearchivist-companion/) and [Chrome](https://chrome.google.com/webstore/detail/tubearchivist-companion/jjnkmicfnfojkkgobdfeieblocadmcie)
|
- [Browser Extension](https://github.com/tubearchivist/browser-extension) Tube Archivist Companion, for [Firefox](https://addons.mozilla.org/addon/tubearchivist-companion/) and [Chrome](https://chrome.google.com/webstore/detail/tubearchivist-companion/jjnkmicfnfojkkgobdfeieblocadmcie)
|
||||||
- [Tube Archivist Metrics](https://github.com/tubearchivist/tubearchivist-metrics) to create statistics in Prometheus/OpenMetrics format.
|
- [Jellyfin Plugin](https://github.com/tubearchivist/tubearchivist-jf-plugin): Add your videos to Jellyfin
|
||||||
- [Showcase](SHOWCASE.MD) To see more screenshots and Youtube videos showing off TubeArchvist.
|
- [Plex Plugin](https://github.com/tubearchivist/tubearchivist-plex): Add your videos to Plex
|
||||||
|
|
||||||
## Installing
|
## Installing
|
||||||
For minimal system requirements, the Tube Archivist stack needs around 2GB of available memory for a small testing setup and around 4GB of available memory for a mid to large sized installation. Minimal with dual core with 4 threads, better quad core plus.
|
For minimal system requirements, the Tube Archivist stack needs around 2GB of available memory for a small testing setup and around 4GB of available memory for a mid to large sized installation. Minimal with dual core with 4 threads, better quad core plus.
|
||||||
|
@ -60,7 +58,12 @@ Take a look at the example [docker-compose.yml](https://github.com/tubearchivist
|
||||||
| TZ | Set your timezone for the scheduler | Required |
|
| TZ | Set your timezone for the scheduler | Required |
|
||||||
| TA_PORT | Overwrite Nginx port | Optional |
|
| TA_PORT | Overwrite Nginx port | Optional |
|
||||||
| TA_UWSGI_PORT | Overwrite container internal uwsgi port | Optional |
|
| TA_UWSGI_PORT | Overwrite container internal uwsgi port | Optional |
|
||||||
|
| TA_ENABLE_AUTH_PROXY | Enables support for forwarding auth in reverse proxies | [Read more](https://docs.tubearchivist.com/configuration/forward-auth/) |
|
||||||
|
| TA_AUTH_PROXY_USERNAME_HEADER | Header containing username to log in | Optional |
|
||||||
|
| TA_AUTH_PROXY_LOGOUT_URL | Logout URL for forwarded auth | Optional |
|
||||||
| ES_URL | URL That ElasticSearch runs on | Optional |
|
| ES_URL | URL That ElasticSearch runs on | Optional |
|
||||||
|
| ES_DISABLE_VERIFY_SSL | Disable ElasticSearch SSL certificate verification | Optional |
|
||||||
|
| ES_SNAPSHOT_DIR | Custom path where elastic search stores snapshots for master/data nodes | Optional |
|
||||||
| HOST_GID | Allow TA to own the video files instead of container user | Optional |
|
| HOST_GID | Allow TA to own the video files instead of container user | Optional |
|
||||||
| HOST_UID | Allow TA to own the video files instead of container user | Optional |
|
| HOST_UID | Allow TA to own the video files instead of container user | Optional |
|
||||||
| ELASTIC_USER | Change the default ElasticSearch user | Optional |
|
| ELASTIC_USER | Change the default ElasticSearch user | Optional |
|
||||||
|
@ -79,7 +82,7 @@ Take a look at the example [docker-compose.yml](https://github.com/tubearchivist
|
||||||
## Update
|
## Update
|
||||||
Always use the *latest* (the default) or a named semantic version tag for the docker images. The *unstable* tags are only for your testing environment, there might not be an update path for these testing builds.
|
Always use the *latest* (the default) or a named semantic version tag for the docker images. The *unstable* tags are only for your testing environment, there might not be an update path for these testing builds.
|
||||||
|
|
||||||
You will see the current version number of **Tube Archivist** in the footer of the interface. There is a daily version check task querying tubearchivist.com, notifying you of any new releases in the footer. To take advantage of the latest fixes and improvements, make sure you are running the *latest and greatest*. After updating, check the footer to verify you are running the expected version.
|
You will see the current version number of **Tube Archivist** in the footer of the interface. There is a daily version check task querying tubearchivist.com, notifying you of any new releases in the footer. To update, you need to update the docker images, the method for which will depend on your platform. For example, if you're using `docker-compose`, run `docker-compose pull` and then restart with `docker-compose up -d`. After updating, check the footer to verify you are running the expected version.
|
||||||
|
|
||||||
- This project is tested for updates between one or two releases maximum. Further updates back may or may not be supported and you might have to reset your index and configurations to update. Ideally apply new updates at least once per month.
|
- This project is tested for updates between one or two releases maximum. Further updates back may or may not be supported and you might have to reset your index and configurations to update. Ideally apply new updates at least once per month.
|
||||||
- There can be breaking changes between updates, particularly as the application grows, new environment variables or settings might be required for you to set in the your docker-compose file. *Always* check the **release notes**: Any breaking changes will be marked there.
|
- There can be breaking changes between updates, particularly as the application grows, new environment variables or settings might be required for you to set in the your docker-compose file. *Always* check the **release notes**: Any breaking changes will be marked there.
|
||||||
|
@ -132,28 +135,36 @@ The Elasticsearch index will turn to ***read only*** if the disk usage of the co
|
||||||
|
|
||||||
Similar to that, TubeArchivist will become all sorts of messed up when running out of disk space. There are some error messages in the logs when that happens, but it's best to make sure to have enough disk space before starting to download.
|
Similar to that, TubeArchivist will become all sorts of messed up when running out of disk space. There are some error messages in the logs when that happens, but it's best to make sure to have enough disk space before starting to download.
|
||||||
|
|
||||||
|
## `error setting rlimit`
|
||||||
|
If you are seeing errors like `failed to create shim: OCI runtime create failed` and `error during container init: error setting rlimits`, this means docker can't set these limits, usually because they are set at another place or are incompatible because of other reasons. Solution is to remove the `ulimits` key from the ES container in your docker compose and start again.
|
||||||
|
|
||||||
|
This can happen if you have nested virtualizations, e.g. LXC running Docker in Proxmox.
|
||||||
|
|
||||||
## Known limitations
|
## Known limitations
|
||||||
- Video files created by Tube Archivist need to be playable in your browser of choice. Not every codec is compatible with every browser and might require some testing with format selection.
|
- Video files created by Tube Archivist need to be playable in your browser of choice. Not every codec is compatible with every browser and might require some testing with format selection.
|
||||||
- Every limitation of **yt-dlp** will also be present in Tube Archivist. If **yt-dlp** can't download or extract a video for any reason, Tube Archivist won't be able to either.
|
- Every limitation of **yt-dlp** will also be present in Tube Archivist. If **yt-dlp** can't download or extract a video for any reason, Tube Archivist won't be able to either.
|
||||||
- There is currently no flexibility in naming of the media files.
|
- There is no flexibility in naming of the media files.
|
||||||
|
|
||||||
## Roadmap
|
## Roadmap
|
||||||
We have come far, nonetheless we are not short of ideas on how to improve and extend this project. Issues waiting for you to be tackled in no particular order:
|
We have come far, nonetheless we are not short of ideas on how to improve and extend this project. Issues waiting for you to be tackled in no particular order:
|
||||||
|
|
||||||
- [ ] User roles
|
- [ ] User roles
|
||||||
|
- [ ] Audio download
|
||||||
- [ ] Podcast mode to serve channel as mp3
|
- [ ] Podcast mode to serve channel as mp3
|
||||||
- [ ] Implement [PyFilesystem](https://github.com/PyFilesystem/pyfilesystem2) for flexible video storage
|
- [ ] Random and repeat controls ([#108](https://github.com/tubearchivist/tubearchivist/issues/108), [#220](https://github.com/tubearchivist/tubearchivist/issues/220))
|
||||||
- [ ] Implement [Apprise](https://github.com/caronc/apprise) for notifications ([#97](https://github.com/tubearchivist/tubearchivist/issues/97))
|
|
||||||
- [ ] User created playlists, random and repeat controls ([#108](https://github.com/tubearchivist/tubearchivist/issues/108), [#220](https://github.com/tubearchivist/tubearchivist/issues/220))
|
|
||||||
- [ ] Auto play or play next link ([#226](https://github.com/tubearchivist/tubearchivist/issues/226))
|
- [ ] Auto play or play next link ([#226](https://github.com/tubearchivist/tubearchivist/issues/226))
|
||||||
- [ ] Multi language support
|
- [ ] Multi language support
|
||||||
- [ ] Show total video downloaded vs total videos available in channel
|
- [ ] Show total video downloaded vs total videos available in channel
|
||||||
- [ ] Add statistics of index
|
|
||||||
- [ ] Download speed schedule ([#198](https://github.com/tubearchivist/tubearchivist/issues/198))
|
|
||||||
- [ ] Download or Ignore videos by keyword ([#163](https://github.com/tubearchivist/tubearchivist/issues/163))
|
- [ ] Download or Ignore videos by keyword ([#163](https://github.com/tubearchivist/tubearchivist/issues/163))
|
||||||
- [ ] Custom searchable notes to videos, channels, playlists ([#144](https://github.com/tubearchivist/tubearchivist/issues/144))
|
- [ ] Custom searchable notes to videos, channels, playlists ([#144](https://github.com/tubearchivist/tubearchivist/issues/144))
|
||||||
|
- [ ] Search comments
|
||||||
|
- [ ] Search download queue
|
||||||
|
- [ ] Configure shorts, streams and video sizes per channel
|
||||||
|
|
||||||
Implemented:
|
Implemented:
|
||||||
|
- [X] User created playlists [2024-04-10]
|
||||||
|
- [X] Add statistics of index [2023-09-03]
|
||||||
|
- [X] Implement [Apprise](https://github.com/caronc/apprise) for notifications [2023-08-05]
|
||||||
- [X] Download video comments [2022-11-30]
|
- [X] Download video comments [2022-11-30]
|
||||||
- [X] Show similar videos on video page [2022-11-30]
|
- [X] Show similar videos on video page [2022-11-30]
|
||||||
- [X] Implement complete offline media file import from json file [2022-08-20]
|
- [X] Implement complete offline media file import from json file [2022-08-20]
|
||||||
|
@ -177,6 +188,19 @@ Implemented:
|
||||||
- [X] Backup and restore [2021-09-22]
|
- [X] Backup and restore [2021-09-22]
|
||||||
- [X] Scan your file system to index already downloaded videos [2021-09-14]
|
- [X] Scan your file system to index already downloaded videos [2021-09-14]
|
||||||
|
|
||||||
|
## User Scripts
|
||||||
|
This is a list of useful user scripts, generously created from folks like you to extend this project and its functionality. Make sure to check the respective repository links for detailed license information.
|
||||||
|
|
||||||
|
This is your time to shine, [read this](https://github.com/tubearchivist/tubearchivist/blob/master/CONTRIBUTING.md#user-scripts) then open a PR to add your script here.
|
||||||
|
|
||||||
|
- [danieljue/ta_dl_page_script](https://github.com/danieljue/ta_dl_page_script): Helper browser script to prioritize a channels' videos in download queue.
|
||||||
|
- [dot-mike/ta-scripts](https://github.com/dot-mike/ta-scripts): A collection of personal scripts for managing TubeArchivist.
|
||||||
|
- [DarkFighterLuke/ta_base_url_nginx](https://gist.github.com/DarkFighterLuke/4561b6bfbf83720493dc59171c58ac36): Set base URL with Nginx when you can't use subdomains.
|
||||||
|
- [lamusmaser/ta_migration_helper](https://github.com/lamusmaser/ta_migration_helper): Advanced helper script for migration issues to TubeArchivist v0.4.4 or later.
|
||||||
|
- [lamusmaser/create_info_json](https://gist.github.com/lamusmaser/837fb58f73ea0cad784a33497932e0dd): Script to generate `.info.json` files using `ffmpeg` collecting information from downloaded videos.
|
||||||
|
- [lamusmaser/ta_fix_for_video_redirection](https://github.com/lamusmaser/ta_fix_for_video_redirection): Script to fix videos that were incorrectly indexed by YouTube's "Video is Unavailable" response.
|
||||||
|
- [RoninTech/ta-helper](https://github.com/RoninTech/ta-helper): Helper script to provide a symlink association to reference TubeArchivist videos with their original titles.
|
||||||
|
|
||||||
## Donate
|
## Donate
|
||||||
The best donation to **Tube Archivist** is your time, take a look at the [contribution page](CONTRIBUTING.md) to get started.
|
The best donation to **Tube Archivist** is your time, take a look at the [contribution page](CONTRIBUTING.md) to get started.
|
||||||
Second best way to support the development is to provide for caffeinated beverages:
|
Second best way to support the development is to provide for caffeinated beverages:
|
||||||
|
@ -187,6 +211,8 @@ Second best way to support the development is to provide for caffeinated beverag
|
||||||
|
|
||||||
## Notable mentions
|
## Notable mentions
|
||||||
This is a selection of places where this project has been featured on reddit, in the news, blogs or any other online media, newest on top.
|
This is a selection of places where this project has been featured on reddit, in the news, blogs or any other online media, newest on top.
|
||||||
|
* **ycombinator**: Tube Archivist on Hackernews front page, [2023-07-16][[link](https://news.ycombinator.com/item?id=36744395)]
|
||||||
|
* **linux-community.de**: Tube Archivist bringt Ordnung in die Youtube-Sammlung, [German][2023-05-01][[link](https://www.linux-community.de/ausgaben/linuxuser/2023/05/tube-archivist-bringt-ordnung-in-die-youtube-sammlung/)]
|
||||||
* **noted.lol**: Dev Debrief, An Interview With the Developer of Tube Archivist, [2023-03-30] [[link](https://noted.lol/dev-debrief-tube-archivist/)]
|
* **noted.lol**: Dev Debrief, An Interview With the Developer of Tube Archivist, [2023-03-30] [[link](https://noted.lol/dev-debrief-tube-archivist/)]
|
||||||
* **console.substack.com**: Interview With Simon of Tube Archivist, [2023-01-29] [[link](https://console.substack.com/p/console-142#%C2%A7interview-with-simon-of-tube-archivist)]
|
* **console.substack.com**: Interview With Simon of Tube Archivist, [2023-01-29] [[link](https://console.substack.com/p/console-142#%C2%A7interview-with-simon-of-tube-archivist)]
|
||||||
* **reddit.com**: Tube Archivist v0.3.0 - Now Archiving Comments, [2022-12-02] [[link](https://www.reddit.com/r/selfhosted/comments/zaonzp/tube_archivist_v030_now_archiving_comments/)]
|
* **reddit.com**: Tube Archivist v0.3.0 - Now Archiving Comments, [2022-12-02] [[link](https://www.reddit.com/r/selfhosted/comments/zaonzp/tube_archivist_v030_now_archiving_comments/)]
|
||||||
|
|
18
SHOWCASE.MD
|
@ -3,17 +3,23 @@
|
||||||
Video featuring Tube Archivist generously created by [IBRACORP](https://www.youtube.com/@IBRACORP).
|
Video featuring Tube Archivist generously created by [IBRACORP](https://www.youtube.com/@IBRACORP).
|
||||||
|
|
||||||
## Screenshots
|
## Screenshots
|
||||||
![home screenshot](assets/tube-archivist-screenshot-home.png?raw=true "Tube Archivist Home")
|
![login screenshot](assets/tube-archivist-login.png?raw=true "Tube Archivist Login")
|
||||||
|
*Login Page*: Secure way to access your media collection.
|
||||||
|
|
||||||
|
![home screenshot](assets/tube-archivist-home.png?raw=true "Tube Archivist Home")
|
||||||
*Home Page*: Your recent videos, continue watching incomplete videos.
|
*Home Page*: Your recent videos, continue watching incomplete videos.
|
||||||
|
|
||||||
![channels screenshot](assets/tube-archivist-screenshot-channels.png?raw=true "Tube Archivist Channels")
|
![channels screenshot](assets/tube-archivist-channels.png?raw=true "Tube Archivist Channels")
|
||||||
*All Channels*: A list of all your indexed channels, filtered by subscribed only.
|
*All Channels*: A list of all your indexed channels, filtered by subscribed only.
|
||||||
|
|
||||||
![single channel screenshot](assets/tube-archivist-screenshot-single-channel.png?raw=true "Tube Archivist Single Channel")
|
![single channel screenshot](assets/tube-archivist-single-channel.png?raw=true "Tube Archivist Single Channel")
|
||||||
*Single Channel*: Single channel page with additional metadata and sub pages.
|
*Single Channel*: Single channel page with additional metadata and sub pages.
|
||||||
|
|
||||||
![video page screenshot](assets/tube-archivist-screenshot-video.png?raw=true "Tube Archivist Video Page")
|
![video page screenshot](assets/tube-archivist-video.png?raw=true "Tube Archivist Video Page")
|
||||||
*Video Page*: Stream your video directly from the interface.
|
*Video Page*: Stream your video directly from the interface.
|
||||||
|
|
||||||
![video page screenshot](assets/tube-archivist-screenshot-download.png?raw=true "Tube Archivist Video Page")
|
![video page screenshot](assets/tube-archivist-download.png?raw=true "Tube Archivist Video Page")
|
||||||
*Downloads Page*: Add, control and monitor your download queue.
|
*Downloads Page*: Add, control, and monitor your download queue.
|
||||||
|
|
||||||
|
![search page screenshot](assets/tube-archivist-search.png?raw=true "Tube Archivist Search Page")
|
||||||
|
*Search Page*. Use expressions to quickly search through your collection.
|
||||||
|
|
Before Width: | Height: | Size: 49 KiB |
After Width: | Height: | Size: 516 KiB |
After Width: | Height: | Size: 541 KiB |
After Width: | Height: | Size: 1.6 MiB |
After Width: | Height: | Size: 578 KiB |
After Width: | Height: | Size: 106 KiB |
Before Width: | Height: | Size: 131 KiB |
Before Width: | Height: | Size: 79 KiB |
Before Width: | Height: | Size: 174 KiB |
Before Width: | Height: | Size: 166 KiB |
Before Width: | Height: | Size: 238 KiB |
After Width: | Height: | Size: 96 KiB |
After Width: | Height: | Size: 716 KiB |
After Width: | Height: | Size: 684 KiB |
10
deploy.sh
|
@ -25,6 +25,7 @@ function sync_blackhole {
|
||||||
--exclude ".gitignore" \
|
--exclude ".gitignore" \
|
||||||
--exclude "**/cache" \
|
--exclude "**/cache" \
|
||||||
--exclude "**/__pycache__/" \
|
--exclude "**/__pycache__/" \
|
||||||
|
--exclude ".venv" \
|
||||||
--exclude "db.sqlite3" \
|
--exclude "db.sqlite3" \
|
||||||
--exclude ".mypy_cache" \
|
--exclude ".mypy_cache" \
|
||||||
. -e ssh "$host":tubearchivist
|
. -e ssh "$host":tubearchivist
|
||||||
|
@ -49,6 +50,7 @@ function sync_test {
|
||||||
--exclude ".gitignore" \
|
--exclude ".gitignore" \
|
||||||
--exclude "**/cache" \
|
--exclude "**/cache" \
|
||||||
--exclude "**/__pycache__/" \
|
--exclude "**/__pycache__/" \
|
||||||
|
--exclude ".venv" \
|
||||||
--exclude "db.sqlite3" \
|
--exclude "db.sqlite3" \
|
||||||
--exclude ".mypy_cache" \
|
--exclude ".mypy_cache" \
|
||||||
. -e ssh "$host":tubearchivist
|
. -e ssh "$host":tubearchivist
|
||||||
|
@ -87,14 +89,14 @@ function validate {
|
||||||
# note: this logic is duplicated in the `./github/workflows/lint_python.yml` config
|
# note: this logic is duplicated in the `./github/workflows/lint_python.yml` config
|
||||||
# if you update this file, you should update that as well
|
# if you update this file, you should update that as well
|
||||||
echo "running black"
|
echo "running black"
|
||||||
black --exclude "migrations/*" --diff --color --check -l 79 "$check_path"
|
black --force-exclude "migrations/*" --diff --color --check -l 79 "$check_path"
|
||||||
echo "running codespell"
|
echo "running codespell"
|
||||||
codespell --skip="./.git,./package.json,./package-lock.json,./node_modules,./.mypy_cache" "$check_path"
|
codespell --skip="./.git,./.venv,./package.json,./package-lock.json,./node_modules,./.mypy_cache" "$check_path"
|
||||||
echo "running flake8"
|
echo "running flake8"
|
||||||
flake8 "$check_path" --exclude "migrations" --count --max-complexity=10 \
|
flake8 "$check_path" --exclude "migrations,.venv" --count --max-complexity=10 \
|
||||||
--max-line-length=79 --show-source --statistics
|
--max-line-length=79 --show-source --statistics
|
||||||
echo "running isort"
|
echo "running isort"
|
||||||
isort --skip "migrations" --check-only --diff --profile black -l 79 "$check_path"
|
isort --skip "migrations" --skip ".venv" --check-only --diff --profile black -l 79 "$check_path"
|
||||||
printf " \n> all validations passed\n"
|
printf " \n> all validations passed\n"
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
version: '3.3'
|
version: '3.5'
|
||||||
|
|
||||||
services:
|
services:
|
||||||
tubearchivist:
|
tubearchivist:
|
||||||
|
@ -20,6 +20,12 @@ services:
|
||||||
- TA_PASSWORD=verysecret # your initial TA credentials
|
- TA_PASSWORD=verysecret # your initial TA credentials
|
||||||
- ELASTIC_PASSWORD=verysecret # set password for Elasticsearch
|
- ELASTIC_PASSWORD=verysecret # set password for Elasticsearch
|
||||||
- TZ=America/New_York # set your time zone
|
- TZ=America/New_York # set your time zone
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
|
||||||
|
interval: 2m
|
||||||
|
timeout: 10s
|
||||||
|
retries: 3
|
||||||
|
start_period: 30s
|
||||||
depends_on:
|
depends_on:
|
||||||
- archivist-es
|
- archivist-es
|
||||||
- archivist-redis
|
- archivist-redis
|
||||||
|
@ -34,7 +40,7 @@ services:
|
||||||
depends_on:
|
depends_on:
|
||||||
- archivist-es
|
- archivist-es
|
||||||
archivist-es:
|
archivist-es:
|
||||||
image: bbilly1/tubearchivist-es # only for amd64, or use official es 8.7.0
|
image: bbilly1/tubearchivist-es # only for amd64, or use official es 8.13.2
|
||||||
container_name: archivist-es
|
container_name: archivist-es
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
environment:
|
environment:
|
||||||
|
|
|
@ -0,0 +1,71 @@
|
||||||
|
"""
|
||||||
|
ffmpeg link builder
|
||||||
|
copied as into build step in Dockerfile
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import tarfile
|
||||||
|
import urllib.request
|
||||||
|
from enum import Enum
|
||||||
|
|
||||||
|
API_URL = "https://api.github.com/repos/yt-dlp/FFmpeg-Builds/releases/latest"
|
||||||
|
BINARIES = ["ffmpeg", "ffprobe"]
|
||||||
|
|
||||||
|
|
||||||
|
class PlatformFilter(Enum):
|
||||||
|
"""options"""
|
||||||
|
|
||||||
|
ARM64 = "linuxarm64"
|
||||||
|
AMD64 = "linux64"
|
||||||
|
|
||||||
|
|
||||||
|
def get_assets():
|
||||||
|
"""get all available assets from latest build"""
|
||||||
|
with urllib.request.urlopen(API_URL) as f:
|
||||||
|
all_links = json.loads(f.read().decode("utf-8"))
|
||||||
|
|
||||||
|
return all_links
|
||||||
|
|
||||||
|
|
||||||
|
def pick_url(all_links, platform):
|
||||||
|
"""pick url for platform"""
|
||||||
|
filter_by = PlatformFilter[platform.split("/")[1].upper()].value
|
||||||
|
options = [i for i in all_links["assets"] if filter_by in i["name"]]
|
||||||
|
if not options:
|
||||||
|
raise ValueError(f"no valid asset found for filter {filter_by}")
|
||||||
|
|
||||||
|
url_pick = options[0]["browser_download_url"]
|
||||||
|
|
||||||
|
return url_pick
|
||||||
|
|
||||||
|
|
||||||
|
def download_extract(url):
|
||||||
|
"""download and extract binaries"""
|
||||||
|
print("download file")
|
||||||
|
filename, _ = urllib.request.urlretrieve(url)
|
||||||
|
print("extract file")
|
||||||
|
with tarfile.open(filename, "r:xz") as tar:
|
||||||
|
for member in tar.getmembers():
|
||||||
|
member.name = os.path.basename(member.name)
|
||||||
|
if member.name in BINARIES:
|
||||||
|
print(f"extract {member.name}")
|
||||||
|
tar.extract(member, member.name)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
"""entry point"""
|
||||||
|
args = sys.argv
|
||||||
|
if len(args) == 1:
|
||||||
|
platform = "linux/amd64"
|
||||||
|
else:
|
||||||
|
platform = args[1]
|
||||||
|
|
||||||
|
all_links = get_assets()
|
||||||
|
url = pick_url(all_links, platform)
|
||||||
|
download_extract(url)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
|
@ -17,7 +17,7 @@ python manage.py ta_startup
|
||||||
|
|
||||||
# start all tasks
|
# start all tasks
|
||||||
nginx &
|
nginx &
|
||||||
celery -A home.tasks worker --loglevel=INFO &
|
celery -A home.celery worker --loglevel=INFO --max-tasks-per-child 10 &
|
||||||
celery -A home beat --loglevel=INFO \
|
celery -A home beat --loglevel=INFO \
|
||||||
-s "${BEAT_SCHEDULE_PATH:-${cachedir}/celerybeat-schedule}" &
|
--scheduler django_celery_beat.schedulers:DatabaseScheduler &
|
||||||
uwsgi --ini uwsgi.ini
|
uwsgi --ini uwsgi.ini
|
||||||
|
|
|
@ -1,3 +0,0 @@
|
||||||
All user documentation has moved to a more flexible, easier to extend and modify documentation platform accessible [here](https://docs.tubearchivist.com) and built from [here](https://github.com/tubearchivist/docs). Don't make any more changes here, keeping this around for some time to keep old links alive.
|
|
||||||
|
|
||||||
Equivalent Channels page is: [here](https://docs.tubearchivist.com/channels/).
|
|
|
@ -1,3 +0,0 @@
|
||||||
All user documentation has moved to a more flexible, easier to extend and modify documentation platform accessible [here](https://docs.tubearchivist.com) and built from [here](https://github.com/tubearchivist/docs). Don't make any more changes here, keeping this around for some time to keep old links alive.
|
|
||||||
|
|
||||||
Equivalent Downloads page is: [here](https://docs.tubearchivist.com/downloads/).
|
|
|
@ -1,3 +0,0 @@
|
||||||
All user documentation has moved to a more flexible, easier to extend and modify documentation platform accessible [here](https://docs.tubearchivist.com) and built from [here](https://github.com/tubearchivist/docs). Don't make any more changes here, keeping this around for some time to keep old links alive.
|
|
||||||
|
|
||||||
Equivalent FAQ page is: [here](https://docs.tubearchivist.com/faq/).
|
|
|
@ -1 +0,0 @@
|
||||||
All user documentation has moved to a more flexible, easier to extend and modify documentation platform accessible [here](https://docs.tubearchivist.com) and built from [here](https://github.com/tubearchivist/docs). Don't make any more changes here, keeping this around for some time to keep old links alive.
|
|
|
@ -1,3 +0,0 @@
|
||||||
All user documentation has moved to a more flexible, easier to extend and modify documentation platform accessible [here](https://docs.tubearchivist.com) and built from [here](https://github.com/tubearchivist/docs). Don't make any more changes here, keeping this around for some time to keep old links alive.
|
|
||||||
|
|
||||||
Equivalent pages are located under *installation* on the left.
|
|
|
@ -1,3 +0,0 @@
|
||||||
All user documentation has moved to a more flexible, easier to extend and modify documentation platform accessible [here](https://docs.tubearchivist.com) and built from [here](https://github.com/tubearchivist/docs). Don't make any more changes here, keeping this around for some time to keep old links alive.
|
|
||||||
|
|
||||||
Equivalent Playlist page is: [here](https://docs.tubearchivist.com/playlists/).
|
|
|
@ -1,3 +0,0 @@
|
||||||
All user documentation has moved to a more flexible, easier to extend and modify documentation platform accessible [here](https://docs.tubearchivist.com) and built from [here](https://github.com/tubearchivist/docs). Don't make any more changes here, keeping this around for some time to keep old links alive.
|
|
||||||
|
|
||||||
Equivalent Search page is: [here](https://docs.tubearchivist.com/search/).
|
|
|
@ -1,3 +0,0 @@
|
||||||
All user documentation has moved to a more flexible, easier to extend and modify documentation platform accessible [here](https://docs.tubearchivist.com) and built from [here](https://github.com/tubearchivist/docs). Don't make any more changes here, keeping this around for some time to keep old links alive.
|
|
||||||
|
|
||||||
Equivalent Settings page is: [here](https://docs.tubearchivist.com/settings/).
|
|
|
@ -1,3 +0,0 @@
|
||||||
All user documentation has moved to a more flexible, easier to extend and modify documentation platform accessible [here](https://docs.tubearchivist.com) and built from [here](https://github.com/tubearchivist/docs). Don't make any more changes here, keeping this around for some time to keep old links alive.
|
|
||||||
|
|
||||||
Equivalent Users page is: [here](https://docs.tubearchivist.com/users/).
|
|
|
@ -1,3 +0,0 @@
|
||||||
All user documentation has moved to a more flexible, easier to extend and modify documentation platform accessible [here](https://docs.tubearchivist.com) and built from [here](https://github.com/tubearchivist/docs). Don't make any more changes here, keeping this around for some time to keep old links alive.
|
|
||||||
|
|
||||||
Equivalent Video page is: [here](https://docs.tubearchivist.com/video/).
|
|
|
@ -0,0 +1,351 @@
|
||||||
|
"""aggregations"""
|
||||||
|
|
||||||
|
from home.src.es.connect import ElasticWrap
|
||||||
|
from home.src.ta.helper import get_duration_str
|
||||||
|
from home.src.ta.settings import EnvironmentSettings
|
||||||
|
|
||||||
|
|
||||||
|
class AggBase:
|
||||||
|
"""base class for aggregation calls"""
|
||||||
|
|
||||||
|
path: str = ""
|
||||||
|
data: dict = {}
|
||||||
|
name: str = ""
|
||||||
|
|
||||||
|
def get(self):
|
||||||
|
"""make get call"""
|
||||||
|
response, _ = ElasticWrap(self.path).get(self.data)
|
||||||
|
print(f"[agg][{self.name}] took {response.get('took')} ms to process")
|
||||||
|
|
||||||
|
return response.get("aggregations")
|
||||||
|
|
||||||
|
def process(self):
|
||||||
|
"""implement in subclassess"""
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
|
||||||
|
class Video(AggBase):
|
||||||
|
"""get video stats"""
|
||||||
|
|
||||||
|
name = "video_stats"
|
||||||
|
path = "ta_video/_search"
|
||||||
|
data = {
|
||||||
|
"size": 0,
|
||||||
|
"aggs": {
|
||||||
|
"video_type": {
|
||||||
|
"terms": {"field": "vid_type"},
|
||||||
|
"aggs": {
|
||||||
|
"media_size": {"sum": {"field": "media_size"}},
|
||||||
|
"duration": {"sum": {"field": "player.duration"}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"video_active": {
|
||||||
|
"terms": {"field": "active"},
|
||||||
|
"aggs": {
|
||||||
|
"media_size": {"sum": {"field": "media_size"}},
|
||||||
|
"duration": {"sum": {"field": "player.duration"}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"video_media_size": {"sum": {"field": "media_size"}},
|
||||||
|
"video_count": {"value_count": {"field": "youtube_id"}},
|
||||||
|
"duration": {"sum": {"field": "player.duration"}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
def process(self):
|
||||||
|
"""process aggregation"""
|
||||||
|
aggregations = self.get()
|
||||||
|
|
||||||
|
duration = int(aggregations["duration"]["value"])
|
||||||
|
response = {
|
||||||
|
"doc_count": aggregations["video_count"]["value"],
|
||||||
|
"media_size": int(aggregations["video_media_size"]["value"]),
|
||||||
|
"duration": duration,
|
||||||
|
"duration_str": get_duration_str(duration),
|
||||||
|
}
|
||||||
|
for bucket in aggregations["video_type"]["buckets"]:
|
||||||
|
duration = int(bucket["duration"].get("value"))
|
||||||
|
response.update(
|
||||||
|
{
|
||||||
|
f"type_{bucket['key']}": {
|
||||||
|
"doc_count": bucket.get("doc_count"),
|
||||||
|
"media_size": int(bucket["media_size"].get("value")),
|
||||||
|
"duration": duration,
|
||||||
|
"duration_str": get_duration_str(duration),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
for bucket in aggregations["video_active"]["buckets"]:
|
||||||
|
duration = int(bucket["duration"].get("value"))
|
||||||
|
response.update(
|
||||||
|
{
|
||||||
|
f"active_{bucket['key_as_string']}": {
|
||||||
|
"doc_count": bucket.get("doc_count"),
|
||||||
|
"media_size": int(bucket["media_size"].get("value")),
|
||||||
|
"duration": duration,
|
||||||
|
"duration_str": get_duration_str(duration),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
class Channel(AggBase):
|
||||||
|
"""get channel stats"""
|
||||||
|
|
||||||
|
name = "channel_stats"
|
||||||
|
path = "ta_channel/_search"
|
||||||
|
data = {
|
||||||
|
"size": 0,
|
||||||
|
"aggs": {
|
||||||
|
"channel_count": {"value_count": {"field": "channel_id"}},
|
||||||
|
"channel_active": {"terms": {"field": "channel_active"}},
|
||||||
|
"channel_subscribed": {"terms": {"field": "channel_subscribed"}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
def process(self):
|
||||||
|
"""process aggregation"""
|
||||||
|
aggregations = self.get()
|
||||||
|
|
||||||
|
response = {
|
||||||
|
"doc_count": aggregations["channel_count"].get("value"),
|
||||||
|
}
|
||||||
|
for bucket in aggregations["channel_active"]["buckets"]:
|
||||||
|
key = f"active_{bucket['key_as_string']}"
|
||||||
|
response.update({key: bucket.get("doc_count")})
|
||||||
|
for bucket in aggregations["channel_subscribed"]["buckets"]:
|
||||||
|
key = f"subscribed_{bucket['key_as_string']}"
|
||||||
|
response.update({key: bucket.get("doc_count")})
|
||||||
|
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
class Playlist(AggBase):
|
||||||
|
"""get playlist stats"""
|
||||||
|
|
||||||
|
name = "playlist_stats"
|
||||||
|
path = "ta_playlist/_search"
|
||||||
|
data = {
|
||||||
|
"size": 0,
|
||||||
|
"aggs": {
|
||||||
|
"playlist_count": {"value_count": {"field": "playlist_id"}},
|
||||||
|
"playlist_active": {"terms": {"field": "playlist_active"}},
|
||||||
|
"playlist_subscribed": {"terms": {"field": "playlist_subscribed"}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
def process(self):
|
||||||
|
"""process aggregation"""
|
||||||
|
aggregations = self.get()
|
||||||
|
response = {"doc_count": aggregations["playlist_count"].get("value")}
|
||||||
|
for bucket in aggregations["playlist_active"]["buckets"]:
|
||||||
|
key = f"active_{bucket['key_as_string']}"
|
||||||
|
response.update({key: bucket.get("doc_count")})
|
||||||
|
for bucket in aggregations["playlist_subscribed"]["buckets"]:
|
||||||
|
key = f"subscribed_{bucket['key_as_string']}"
|
||||||
|
response.update({key: bucket.get("doc_count")})
|
||||||
|
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
class Download(AggBase):
|
||||||
|
"""get downloads queue stats"""
|
||||||
|
|
||||||
|
name = "download_queue_stats"
|
||||||
|
path = "ta_download/_search"
|
||||||
|
data = {
|
||||||
|
"size": 0,
|
||||||
|
"aggs": {
|
||||||
|
"status": {"terms": {"field": "status"}},
|
||||||
|
"video_type": {
|
||||||
|
"filter": {"term": {"status": "pending"}},
|
||||||
|
"aggs": {"type_pending": {"terms": {"field": "vid_type"}}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
def process(self):
|
||||||
|
"""process aggregation"""
|
||||||
|
aggregations = self.get()
|
||||||
|
response = {}
|
||||||
|
for bucket in aggregations["status"]["buckets"]:
|
||||||
|
response.update({bucket["key"]: bucket.get("doc_count")})
|
||||||
|
|
||||||
|
for bucket in aggregations["video_type"]["type_pending"]["buckets"]:
|
||||||
|
key = f"pending_{bucket['key']}"
|
||||||
|
response.update({key: bucket.get("doc_count")})
|
||||||
|
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
class WatchProgress(AggBase):
|
||||||
|
"""get watch progress"""
|
||||||
|
|
||||||
|
name = "watch_progress"
|
||||||
|
path = "ta_video/_search"
|
||||||
|
data = {
|
||||||
|
"size": 0,
|
||||||
|
"aggs": {
|
||||||
|
name: {
|
||||||
|
"terms": {"field": "player.watched"},
|
||||||
|
"aggs": {
|
||||||
|
"watch_docs": {
|
||||||
|
"filter": {"terms": {"player.watched": [True, False]}},
|
||||||
|
"aggs": {
|
||||||
|
"true_count": {"value_count": {"field": "_index"}},
|
||||||
|
"duration": {"sum": {"field": "player.duration"}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"total_duration": {"sum": {"field": "player.duration"}},
|
||||||
|
"total_vids": {"value_count": {"field": "_index"}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
def process(self):
|
||||||
|
"""make the call"""
|
||||||
|
aggregations = self.get()
|
||||||
|
buckets = aggregations[self.name]["buckets"]
|
||||||
|
|
||||||
|
response = {}
|
||||||
|
all_duration = int(aggregations["total_duration"].get("value"))
|
||||||
|
response.update(
|
||||||
|
{
|
||||||
|
"total": {
|
||||||
|
"duration": all_duration,
|
||||||
|
"duration_str": get_duration_str(all_duration),
|
||||||
|
"items": aggregations["total_vids"].get("value"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
for bucket in buckets:
|
||||||
|
response.update(self._build_bucket(bucket, all_duration))
|
||||||
|
|
||||||
|
return response
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _build_bucket(bucket, all_duration):
|
||||||
|
"""parse bucket"""
|
||||||
|
|
||||||
|
duration = int(bucket["watch_docs"]["duration"]["value"])
|
||||||
|
duration_str = get_duration_str(duration)
|
||||||
|
items = bucket["watch_docs"]["true_count"]["value"]
|
||||||
|
if bucket["key_as_string"] == "false":
|
||||||
|
key = "unwatched"
|
||||||
|
else:
|
||||||
|
key = "watched"
|
||||||
|
|
||||||
|
bucket_parsed = {
|
||||||
|
key: {
|
||||||
|
"duration": duration,
|
||||||
|
"duration_str": duration_str,
|
||||||
|
"progress": duration / all_duration if all_duration else 0,
|
||||||
|
"items": items,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return bucket_parsed
|
||||||
|
|
||||||
|
|
||||||
|
class DownloadHist(AggBase):
|
||||||
|
"""get downloads histogram last week"""
|
||||||
|
|
||||||
|
name = "videos_last_week"
|
||||||
|
path = "ta_video/_search"
|
||||||
|
data = {
|
||||||
|
"size": 0,
|
||||||
|
"aggs": {
|
||||||
|
name: {
|
||||||
|
"date_histogram": {
|
||||||
|
"field": "date_downloaded",
|
||||||
|
"calendar_interval": "day",
|
||||||
|
"format": "yyyy-MM-dd",
|
||||||
|
"order": {"_key": "desc"},
|
||||||
|
"time_zone": EnvironmentSettings.TZ,
|
||||||
|
},
|
||||||
|
"aggs": {
|
||||||
|
"total_videos": {"value_count": {"field": "youtube_id"}},
|
||||||
|
"media_size": {"sum": {"field": "media_size"}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"query": {
|
||||||
|
"range": {
|
||||||
|
"date_downloaded": {
|
||||||
|
"gte": "now-7d/d",
|
||||||
|
"time_zone": EnvironmentSettings.TZ,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
def process(self):
|
||||||
|
"""process query"""
|
||||||
|
aggregations = self.get()
|
||||||
|
buckets = aggregations[self.name]["buckets"]
|
||||||
|
|
||||||
|
response = [
|
||||||
|
{
|
||||||
|
"date": i.get("key_as_string"),
|
||||||
|
"count": i.get("doc_count"),
|
||||||
|
"media_size": i["media_size"].get("value"),
|
||||||
|
}
|
||||||
|
for i in buckets
|
||||||
|
]
|
||||||
|
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
class BiggestChannel(AggBase):
|
||||||
|
"""get channel aggregations"""
|
||||||
|
|
||||||
|
def __init__(self, order):
|
||||||
|
self.data["aggs"][self.name]["multi_terms"]["order"] = {order: "desc"}
|
||||||
|
|
||||||
|
name = "channel_stats"
|
||||||
|
path = "ta_video/_search"
|
||||||
|
data = {
|
||||||
|
"size": 0,
|
||||||
|
"aggs": {
|
||||||
|
name: {
|
||||||
|
"multi_terms": {
|
||||||
|
"terms": [
|
||||||
|
{"field": "channel.channel_name.keyword"},
|
||||||
|
{"field": "channel.channel_id"},
|
||||||
|
],
|
||||||
|
"order": {"doc_count": "desc"},
|
||||||
|
},
|
||||||
|
"aggs": {
|
||||||
|
"doc_count": {"value_count": {"field": "_index"}},
|
||||||
|
"duration": {"sum": {"field": "player.duration"}},
|
||||||
|
"media_size": {"sum": {"field": "media_size"}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
order_choices = ["doc_count", "duration", "media_size"]
|
||||||
|
|
||||||
|
def process(self):
|
||||||
|
"""process aggregation, order_by validated in the view"""
|
||||||
|
|
||||||
|
aggregations = self.get()
|
||||||
|
buckets = aggregations[self.name]["buckets"]
|
||||||
|
|
||||||
|
response = [
|
||||||
|
{
|
||||||
|
"id": i["key"][1],
|
||||||
|
"name": i["key"][0].title(),
|
||||||
|
"doc_count": i["doc_count"]["value"],
|
||||||
|
"duration": i["duration"]["value"],
|
||||||
|
"duration_str": get_duration_str(int(i["duration"]["value"])),
|
||||||
|
"media_size": i["media_size"]["value"],
|
||||||
|
}
|
||||||
|
for i in buckets
|
||||||
|
]
|
||||||
|
|
||||||
|
return response
|
|
@ -7,15 +7,14 @@ Functionality:
|
||||||
import urllib.parse
|
import urllib.parse
|
||||||
|
|
||||||
from home.src.download.thumbnails import ThumbManager
|
from home.src.download.thumbnails import ThumbManager
|
||||||
from home.src.ta.config import AppConfig
|
from home.src.ta.helper import date_praser, get_duration_str
|
||||||
from home.src.ta.helper import date_praser
|
from home.src.ta.settings import EnvironmentSettings
|
||||||
|
|
||||||
|
|
||||||
class SearchProcess:
|
class SearchProcess:
|
||||||
"""process search results"""
|
"""process search results"""
|
||||||
|
|
||||||
CONFIG = AppConfig().config
|
CACHE_DIR = EnvironmentSettings.CACHE_DIR
|
||||||
CACHE_DIR = CONFIG["application"]["cache_dir"]
|
|
||||||
|
|
||||||
def __init__(self, response):
|
def __init__(self, response):
|
||||||
self.response = response
|
self.response = response
|
||||||
|
@ -50,6 +49,16 @@ class SearchProcess:
|
||||||
processed = self._process_download(result["_source"])
|
processed = self._process_download(result["_source"])
|
||||||
if index == "ta_comment":
|
if index == "ta_comment":
|
||||||
processed = self._process_comment(result["_source"])
|
processed = self._process_comment(result["_source"])
|
||||||
|
if index == "ta_subtitle":
|
||||||
|
processed = self._process_subtitle(result)
|
||||||
|
|
||||||
|
if isinstance(processed, dict):
|
||||||
|
processed.update(
|
||||||
|
{
|
||||||
|
"_index": index,
|
||||||
|
"_score": round(result.get("_score") or 0, 2),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
return processed
|
return processed
|
||||||
|
|
||||||
|
@ -139,3 +148,29 @@ class SearchProcess:
|
||||||
processed_comments[-1]["comment_replies"].append(comment)
|
processed_comments[-1]["comment_replies"].append(comment)
|
||||||
|
|
||||||
return processed_comments
|
return processed_comments
|
||||||
|
|
||||||
|
def _process_subtitle(self, result):
|
||||||
|
"""take complete result dict to extract highlight"""
|
||||||
|
subtitle_dict = result["_source"]
|
||||||
|
highlight = result.get("highlight")
|
||||||
|
if highlight:
|
||||||
|
# replace lines with the highlighted markdown
|
||||||
|
subtitle_line = highlight.get("subtitle_line")[0]
|
||||||
|
subtitle_dict.update({"subtitle_line": subtitle_line})
|
||||||
|
|
||||||
|
thumb_path = ThumbManager(subtitle_dict["youtube_id"]).vid_thumb_path()
|
||||||
|
subtitle_dict.update({"vid_thumb_url": f"/cache/{thumb_path}"})
|
||||||
|
|
||||||
|
return subtitle_dict
|
||||||
|
|
||||||
|
|
||||||
|
def process_aggs(response):
|
||||||
|
"""convert aggs duration to str"""
|
||||||
|
|
||||||
|
if response.get("aggregations"):
|
||||||
|
aggs = response["aggregations"]
|
||||||
|
if "total_duration" in aggs:
|
||||||
|
duration_sec = int(aggs["total_duration"]["value"])
|
||||||
|
aggs["total_duration"].update(
|
||||||
|
{"value_str": get_duration_str(duration_sec)}
|
||||||
|
)
|
||||||
|
|
|
@ -41,6 +41,11 @@ urlpatterns = [
|
||||||
views.ChannelApiListView.as_view(),
|
views.ChannelApiListView.as_view(),
|
||||||
name="api-channel-list",
|
name="api-channel-list",
|
||||||
),
|
),
|
||||||
|
path(
|
||||||
|
"channel/search/",
|
||||||
|
views.ChannelApiSearchView.as_view(),
|
||||||
|
name="api-channel-search",
|
||||||
|
),
|
||||||
path(
|
path(
|
||||||
"channel/<slug:channel_id>/",
|
"channel/<slug:channel_id>/",
|
||||||
views.ChannelApiView.as_view(),
|
views.ChannelApiView.as_view(),
|
||||||
|
@ -91,6 +96,16 @@ urlpatterns = [
|
||||||
views.SnapshotApiView.as_view(),
|
views.SnapshotApiView.as_view(),
|
||||||
name="api-snapshot",
|
name="api-snapshot",
|
||||||
),
|
),
|
||||||
|
path(
|
||||||
|
"backup/",
|
||||||
|
views.BackupApiListView.as_view(),
|
||||||
|
name="api-backup-list",
|
||||||
|
),
|
||||||
|
path(
|
||||||
|
"backup/<str:filename>/",
|
||||||
|
views.BackupApiView.as_view(),
|
||||||
|
name="api-backup",
|
||||||
|
),
|
||||||
path(
|
path(
|
||||||
"task-name/",
|
"task-name/",
|
||||||
views.TaskListView.as_view(),
|
views.TaskListView.as_view(),
|
||||||
|
@ -106,6 +121,21 @@ urlpatterns = [
|
||||||
views.TaskIDView.as_view(),
|
views.TaskIDView.as_view(),
|
||||||
name="api-task-id",
|
name="api-task-id",
|
||||||
),
|
),
|
||||||
|
path(
|
||||||
|
"schedule/",
|
||||||
|
views.ScheduleView.as_view(),
|
||||||
|
name="api-schedule",
|
||||||
|
),
|
||||||
|
path(
|
||||||
|
"schedule/notification/",
|
||||||
|
views.ScheduleNotification.as_view(),
|
||||||
|
name="api-schedule-notification",
|
||||||
|
),
|
||||||
|
path(
|
||||||
|
"config/user/",
|
||||||
|
views.UserConfigView.as_view(),
|
||||||
|
name="api-config-user",
|
||||||
|
),
|
||||||
path(
|
path(
|
||||||
"cookie/",
|
"cookie/",
|
||||||
views.CookieView.as_view(),
|
views.CookieView.as_view(),
|
||||||
|
@ -131,4 +161,39 @@ urlpatterns = [
|
||||||
views.NotificationView.as_view(),
|
views.NotificationView.as_view(),
|
||||||
name="api-notification",
|
name="api-notification",
|
||||||
),
|
),
|
||||||
|
path(
|
||||||
|
"stats/video/",
|
||||||
|
views.StatVideoView.as_view(),
|
||||||
|
name="api-stats-video",
|
||||||
|
),
|
||||||
|
path(
|
||||||
|
"stats/channel/",
|
||||||
|
views.StatChannelView.as_view(),
|
||||||
|
name="api-stats-channel",
|
||||||
|
),
|
||||||
|
path(
|
||||||
|
"stats/playlist/",
|
||||||
|
views.StatPlaylistView.as_view(),
|
||||||
|
name="api-stats-playlist",
|
||||||
|
),
|
||||||
|
path(
|
||||||
|
"stats/download/",
|
||||||
|
views.StatDownloadView.as_view(),
|
||||||
|
name="api-stats-download",
|
||||||
|
),
|
||||||
|
path(
|
||||||
|
"stats/watch/",
|
||||||
|
views.StatWatchProgress.as_view(),
|
||||||
|
name="api-stats-watch",
|
||||||
|
),
|
||||||
|
path(
|
||||||
|
"stats/downloadhist/",
|
||||||
|
views.StatDownloadHist.as_view(),
|
||||||
|
name="api-stats-downloadhist",
|
||||||
|
),
|
||||||
|
path(
|
||||||
|
"stats/biggestchannels/",
|
||||||
|
views.StatBiggestChannel.as_view(),
|
||||||
|
name="api-stats-biggestchannels",
|
||||||
|
),
|
||||||
]
|
]
|
||||||
|
|
|
@ -1,49 +1,97 @@
|
||||||
"""all API views"""
|
"""all API views"""
|
||||||
|
|
||||||
|
from api.src.aggs import (
|
||||||
|
BiggestChannel,
|
||||||
|
Channel,
|
||||||
|
Download,
|
||||||
|
DownloadHist,
|
||||||
|
Playlist,
|
||||||
|
Video,
|
||||||
|
WatchProgress,
|
||||||
|
)
|
||||||
from api.src.search_processor import SearchProcess
|
from api.src.search_processor import SearchProcess
|
||||||
|
from home.models import CustomPeriodicTask
|
||||||
from home.src.download.queue import PendingInteract
|
from home.src.download.queue import PendingInteract
|
||||||
|
from home.src.download.subscriptions import (
|
||||||
|
ChannelSubscription,
|
||||||
|
PlaylistSubscription,
|
||||||
|
)
|
||||||
from home.src.download.yt_dlp_base import CookieHandler
|
from home.src.download.yt_dlp_base import CookieHandler
|
||||||
|
from home.src.es.backup import ElasticBackup
|
||||||
from home.src.es.connect import ElasticWrap
|
from home.src.es.connect import ElasticWrap
|
||||||
from home.src.es.snapshot import ElasticSnapshot
|
from home.src.es.snapshot import ElasticSnapshot
|
||||||
from home.src.frontend.searching import SearchForm
|
from home.src.frontend.searching import SearchForm
|
||||||
from home.src.frontend.watched import WatchState
|
from home.src.frontend.watched import WatchState
|
||||||
from home.src.index.channel import YoutubeChannel
|
from home.src.index.channel import YoutubeChannel
|
||||||
from home.src.index.generic import Pagination
|
from home.src.index.generic import Pagination
|
||||||
|
from home.src.index.playlist import YoutubePlaylist
|
||||||
from home.src.index.reindex import ReindexProgress
|
from home.src.index.reindex import ReindexProgress
|
||||||
from home.src.index.video import SponsorBlock, YoutubeVideo
|
from home.src.index.video import SponsorBlock, YoutubeVideo
|
||||||
from home.src.ta.config import AppConfig
|
from home.src.ta.config import AppConfig, ReleaseVersion
|
||||||
from home.src.ta.ta_redis import RedisArchivist, RedisQueue
|
from home.src.ta.notify import Notifications, get_all_notifications
|
||||||
|
from home.src.ta.settings import EnvironmentSettings
|
||||||
|
from home.src.ta.ta_redis import RedisArchivist
|
||||||
|
from home.src.ta.task_config import TASK_CONFIG
|
||||||
from home.src.ta.task_manager import TaskCommand, TaskManager
|
from home.src.ta.task_manager import TaskCommand, TaskManager
|
||||||
from home.src.ta.urlparser import Parser
|
from home.src.ta.urlparser import Parser
|
||||||
|
from home.src.ta.users import UserConfig
|
||||||
from home.tasks import (
|
from home.tasks import (
|
||||||
BaseTask,
|
|
||||||
check_reindex,
|
check_reindex,
|
||||||
download_pending,
|
download_pending,
|
||||||
extrac_dl,
|
extrac_dl,
|
||||||
|
run_restore_backup,
|
||||||
subscribe_to,
|
subscribe_to,
|
||||||
)
|
)
|
||||||
|
from rest_framework import permissions, status
|
||||||
from rest_framework.authentication import (
|
from rest_framework.authentication import (
|
||||||
SessionAuthentication,
|
SessionAuthentication,
|
||||||
TokenAuthentication,
|
TokenAuthentication,
|
||||||
)
|
)
|
||||||
from rest_framework.authtoken.models import Token
|
from rest_framework.authtoken.models import Token
|
||||||
from rest_framework.authtoken.views import ObtainAuthToken
|
from rest_framework.authtoken.views import ObtainAuthToken
|
||||||
from rest_framework.permissions import IsAuthenticated
|
|
||||||
from rest_framework.response import Response
|
from rest_framework.response import Response
|
||||||
from rest_framework.views import APIView
|
from rest_framework.views import APIView
|
||||||
|
|
||||||
|
|
||||||
|
def check_admin(user):
|
||||||
|
"""check for admin permission for restricted views"""
|
||||||
|
return user.is_staff or user.groups.filter(name="admin").exists()
|
||||||
|
|
||||||
|
|
||||||
|
class AdminOnly(permissions.BasePermission):
|
||||||
|
"""allow only admin"""
|
||||||
|
|
||||||
|
def has_permission(self, request, view):
|
||||||
|
return check_admin(request.user)
|
||||||
|
|
||||||
|
|
||||||
|
class AdminWriteOnly(permissions.BasePermission):
|
||||||
|
"""allow only admin writes"""
|
||||||
|
|
||||||
|
def has_permission(self, request, view):
|
||||||
|
if request.method in permissions.SAFE_METHODS:
|
||||||
|
return permissions.IsAuthenticated().has_permission(request, view)
|
||||||
|
|
||||||
|
return check_admin(request.user)
|
||||||
|
|
||||||
|
|
||||||
class ApiBaseView(APIView):
|
class ApiBaseView(APIView):
|
||||||
"""base view to inherit from"""
|
"""base view to inherit from"""
|
||||||
|
|
||||||
authentication_classes = [SessionAuthentication, TokenAuthentication]
|
authentication_classes = [SessionAuthentication, TokenAuthentication]
|
||||||
permission_classes = [IsAuthenticated]
|
permission_classes = [permissions.IsAuthenticated]
|
||||||
search_base = False
|
search_base = ""
|
||||||
data = False
|
data = ""
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.response = {"data": False, "config": AppConfig().config}
|
self.response = {
|
||||||
|
"data": False,
|
||||||
|
"config": {
|
||||||
|
"enable_cast": EnvironmentSettings.ENABLE_CAST,
|
||||||
|
"downloads": AppConfig().config["downloads"],
|
||||||
|
},
|
||||||
|
}
|
||||||
self.data = {"query": {"match_all": {}}}
|
self.data = {"query": {"match_all": {}}}
|
||||||
self.status_code = False
|
self.status_code = False
|
||||||
self.context = False
|
self.context = False
|
||||||
|
@ -96,6 +144,7 @@ class VideoApiView(ApiBaseView):
|
||||||
"""
|
"""
|
||||||
|
|
||||||
search_base = "ta_video/_doc/"
|
search_base = "ta_video/_doc/"
|
||||||
|
permission_classes = [AdminWriteOnly]
|
||||||
|
|
||||||
def get(self, request, video_id):
|
def get(self, request, video_id):
|
||||||
# pylint: disable=unused-argument
|
# pylint: disable=unused-argument
|
||||||
|
@ -134,7 +183,7 @@ class VideoApiListView(ApiBaseView):
|
||||||
|
|
||||||
|
|
||||||
class VideoProgressView(ApiBaseView):
|
class VideoProgressView(ApiBaseView):
|
||||||
"""resolves to /api/video/<video_id>/
|
"""resolves to /api/video/<video_id>/progress/
|
||||||
handle progress status for video
|
handle progress status for video
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
@ -159,7 +208,6 @@ class VideoProgressView(ApiBaseView):
|
||||||
message = {"position": position, "youtube_id": video_id}
|
message = {"position": position, "youtube_id": video_id}
|
||||||
RedisArchivist().set_message(key, message)
|
RedisArchivist().set_message(key, message)
|
||||||
self.response = request.data
|
self.response = request.data
|
||||||
|
|
||||||
return Response(self.response)
|
return Response(self.response)
|
||||||
|
|
||||||
def delete(self, request, video_id):
|
def delete(self, request, video_id):
|
||||||
|
@ -189,7 +237,7 @@ class VideoCommentView(ApiBaseView):
|
||||||
|
|
||||||
class VideoSimilarView(ApiBaseView):
|
class VideoSimilarView(ApiBaseView):
|
||||||
"""resolves to /api/video/<video-id>/similar/
|
"""resolves to /api/video/<video-id>/similar/
|
||||||
GET: return max 3 videos similar to this
|
GET: return max 6 videos similar to this
|
||||||
"""
|
"""
|
||||||
|
|
||||||
search_base = "ta_video/_search/"
|
search_base = "ta_video/_search/"
|
||||||
|
@ -223,6 +271,10 @@ class VideoSponsorView(ApiBaseView):
|
||||||
# pylint: disable=unused-argument
|
# pylint: disable=unused-argument
|
||||||
|
|
||||||
self.get_document(video_id)
|
self.get_document(video_id)
|
||||||
|
if not self.response.get("data"):
|
||||||
|
message = {"message": "video not found"}
|
||||||
|
return Response(message, status=404)
|
||||||
|
|
||||||
sponsorblock = self.response["data"].get("sponsorblock")
|
sponsorblock = self.response["data"].get("sponsorblock")
|
||||||
|
|
||||||
return Response(sponsorblock)
|
return Response(sponsorblock)
|
||||||
|
@ -266,6 +318,7 @@ class ChannelApiView(ApiBaseView):
|
||||||
"""
|
"""
|
||||||
|
|
||||||
search_base = "ta_channel/_doc/"
|
search_base = "ta_channel/_doc/"
|
||||||
|
permission_classes = [AdminWriteOnly]
|
||||||
|
|
||||||
def get(self, request, channel_id):
|
def get(self, request, channel_id):
|
||||||
# pylint: disable=unused-argument
|
# pylint: disable=unused-argument
|
||||||
|
@ -296,6 +349,7 @@ class ChannelApiListView(ApiBaseView):
|
||||||
|
|
||||||
search_base = "ta_channel/_search/"
|
search_base = "ta_channel/_search/"
|
||||||
valid_filter = ["subscribed"]
|
valid_filter = ["subscribed"]
|
||||||
|
permission_classes = [AdminWriteOnly]
|
||||||
|
|
||||||
def get(self, request):
|
def get(self, request):
|
||||||
"""get request"""
|
"""get request"""
|
||||||
|
@ -318,9 +372,8 @@ class ChannelApiListView(ApiBaseView):
|
||||||
|
|
||||||
return Response(self.response)
|
return Response(self.response)
|
||||||
|
|
||||||
@staticmethod
|
def post(self, request):
|
||||||
def post(request):
|
"""subscribe/unsubscribe to list of channels"""
|
||||||
"""subscribe to list of channels"""
|
|
||||||
data = request.data
|
data = request.data
|
||||||
try:
|
try:
|
||||||
to_add = data["data"]
|
to_add = data["data"]
|
||||||
|
@ -329,12 +382,58 @@ class ChannelApiListView(ApiBaseView):
|
||||||
print(message)
|
print(message)
|
||||||
return Response({"message": message}, status=400)
|
return Response({"message": message}, status=400)
|
||||||
|
|
||||||
pending = [i["channel_id"] for i in to_add if i["channel_subscribed"]]
|
pending = []
|
||||||
url_str = " ".join(pending)
|
for channel_item in to_add:
|
||||||
subscribe_to.delay(url_str)
|
channel_id = channel_item["channel_id"]
|
||||||
|
if channel_item["channel_subscribed"]:
|
||||||
|
pending.append(channel_id)
|
||||||
|
else:
|
||||||
|
self._unsubscribe(channel_id)
|
||||||
|
|
||||||
|
if pending:
|
||||||
|
url_str = " ".join(pending)
|
||||||
|
subscribe_to.delay(url_str, expected_type="channel")
|
||||||
|
|
||||||
return Response(data)
|
return Response(data)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _unsubscribe(channel_id: str):
|
||||||
|
"""unsubscribe"""
|
||||||
|
print(f"[{channel_id}] unsubscribe from channel")
|
||||||
|
ChannelSubscription().change_subscribe(
|
||||||
|
channel_id, channel_subscribed=False
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ChannelApiSearchView(ApiBaseView):
|
||||||
|
"""resolves to /api/channel/search/
|
||||||
|
search for channel
|
||||||
|
"""
|
||||||
|
|
||||||
|
search_base = "ta_channel/_doc/"
|
||||||
|
|
||||||
|
def get(self, request):
|
||||||
|
"""handle get request, search with s parameter"""
|
||||||
|
|
||||||
|
query = request.GET.get("q")
|
||||||
|
if not query:
|
||||||
|
message = "missing expected q parameter"
|
||||||
|
return Response({"message": message, "data": False}, status=400)
|
||||||
|
|
||||||
|
try:
|
||||||
|
parsed = Parser(query).parse()[0]
|
||||||
|
except (ValueError, IndexError, AttributeError):
|
||||||
|
message = f"channel not found: {query}"
|
||||||
|
return Response({"message": message, "data": False}, status=404)
|
||||||
|
|
||||||
|
if not parsed["type"] == "channel":
|
||||||
|
message = "expected type channel"
|
||||||
|
return Response({"message": message, "data": False}, status=400)
|
||||||
|
|
||||||
|
self.get_document(parsed["url"])
|
||||||
|
|
||||||
|
return Response(self.response, status=self.status_code)
|
||||||
|
|
||||||
|
|
||||||
class ChannelApiVideoView(ApiBaseView):
|
class ChannelApiVideoView(ApiBaseView):
|
||||||
"""resolves to /api/channel/<channel-id>/video
|
"""resolves to /api/channel/<channel-id>/video
|
||||||
|
@ -364,15 +463,62 @@ class PlaylistApiListView(ApiBaseView):
|
||||||
"""
|
"""
|
||||||
|
|
||||||
search_base = "ta_playlist/_search/"
|
search_base = "ta_playlist/_search/"
|
||||||
|
permission_classes = [AdminWriteOnly]
|
||||||
|
valid_playlist_type = ["regular", "custom"]
|
||||||
|
|
||||||
def get(self, request):
|
def get(self, request):
|
||||||
"""handle get request"""
|
"""handle get request"""
|
||||||
self.data.update(
|
playlist_type = request.GET.get("playlist_type", None)
|
||||||
{"sort": [{"playlist_name.keyword": {"order": "asc"}}]}
|
query = {"sort": [{"playlist_name.keyword": {"order": "asc"}}]}
|
||||||
)
|
if playlist_type is not None:
|
||||||
|
if playlist_type not in self.valid_playlist_type:
|
||||||
|
message = f"invalid playlist_type {playlist_type}"
|
||||||
|
return Response({"message": message}, status=400)
|
||||||
|
|
||||||
|
query.update(
|
||||||
|
{
|
||||||
|
"query": {
|
||||||
|
"term": {"playlist_type": {"value": playlist_type}}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
self.data.update(query)
|
||||||
self.get_document_list(request)
|
self.get_document_list(request)
|
||||||
return Response(self.response)
|
return Response(self.response)
|
||||||
|
|
||||||
|
def post(self, request):
|
||||||
|
"""subscribe/unsubscribe to list of playlists"""
|
||||||
|
data = request.data
|
||||||
|
try:
|
||||||
|
to_add = data["data"]
|
||||||
|
except KeyError:
|
||||||
|
message = "missing expected data key"
|
||||||
|
print(message)
|
||||||
|
return Response({"message": message}, status=400)
|
||||||
|
|
||||||
|
pending = []
|
||||||
|
for playlist_item in to_add:
|
||||||
|
playlist_id = playlist_item["playlist_id"]
|
||||||
|
if playlist_item["playlist_subscribed"]:
|
||||||
|
pending.append(playlist_id)
|
||||||
|
else:
|
||||||
|
self._unsubscribe(playlist_id)
|
||||||
|
|
||||||
|
if pending:
|
||||||
|
url_str = " ".join(pending)
|
||||||
|
subscribe_to.delay(url_str, expected_type="playlist")
|
||||||
|
|
||||||
|
return Response(data)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _unsubscribe(playlist_id: str):
|
||||||
|
"""unsubscribe"""
|
||||||
|
print(f"[{playlist_id}] unsubscribe from playlist")
|
||||||
|
PlaylistSubscription().change_subscribe(
|
||||||
|
playlist_id, subscribe_status=False
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class PlaylistApiView(ApiBaseView):
|
class PlaylistApiView(ApiBaseView):
|
||||||
"""resolves to /api/playlist/<playlist_id>/
|
"""resolves to /api/playlist/<playlist_id>/
|
||||||
|
@ -380,6 +526,8 @@ class PlaylistApiView(ApiBaseView):
|
||||||
"""
|
"""
|
||||||
|
|
||||||
search_base = "ta_playlist/_doc/"
|
search_base = "ta_playlist/_doc/"
|
||||||
|
permission_classes = [AdminWriteOnly]
|
||||||
|
valid_custom_actions = ["create", "remove", "up", "down", "top", "bottom"]
|
||||||
|
|
||||||
def get(self, request, playlist_id):
|
def get(self, request, playlist_id):
|
||||||
# pylint: disable=unused-argument
|
# pylint: disable=unused-argument
|
||||||
|
@ -387,6 +535,38 @@ class PlaylistApiView(ApiBaseView):
|
||||||
self.get_document(playlist_id)
|
self.get_document(playlist_id)
|
||||||
return Response(self.response, status=self.status_code)
|
return Response(self.response, status=self.status_code)
|
||||||
|
|
||||||
|
def post(self, request, playlist_id):
|
||||||
|
"""post to custom playlist to add a video to list"""
|
||||||
|
playlist = YoutubePlaylist(playlist_id)
|
||||||
|
if not playlist.is_custom_playlist():
|
||||||
|
message = f"playlist with ID {playlist_id} is not custom"
|
||||||
|
return Response({"message": message}, status=400)
|
||||||
|
|
||||||
|
action = request.data.get("action")
|
||||||
|
if action not in self.valid_custom_actions:
|
||||||
|
message = f"invalid action: {action}"
|
||||||
|
return Response({"message": message}, status=400)
|
||||||
|
|
||||||
|
video_id = request.data.get("video_id")
|
||||||
|
if action == "create":
|
||||||
|
playlist.add_video_to_playlist(video_id)
|
||||||
|
else:
|
||||||
|
hide = UserConfig(request.user.id).get_value("hide_watched")
|
||||||
|
playlist.move_video(video_id, action, hide_watched=hide)
|
||||||
|
|
||||||
|
return Response({"success": True}, status=status.HTTP_201_CREATED)
|
||||||
|
|
||||||
|
def delete(self, request, playlist_id):
|
||||||
|
"""delete playlist"""
|
||||||
|
print(f"{playlist_id}: delete playlist")
|
||||||
|
delete_videos = request.GET.get("delete-videos", False)
|
||||||
|
if delete_videos:
|
||||||
|
YoutubePlaylist(playlist_id).delete_videos_playlist()
|
||||||
|
else:
|
||||||
|
YoutubePlaylist(playlist_id).delete_metadata()
|
||||||
|
|
||||||
|
return Response({"success": True})
|
||||||
|
|
||||||
|
|
||||||
class PlaylistApiVideoView(ApiBaseView):
|
class PlaylistApiVideoView(ApiBaseView):
|
||||||
"""resolves to /api/playlist/<playlist_id>/video
|
"""resolves to /api/playlist/<playlist_id>/video
|
||||||
|
@ -414,7 +594,8 @@ class DownloadApiView(ApiBaseView):
|
||||||
"""
|
"""
|
||||||
|
|
||||||
search_base = "ta_download/_doc/"
|
search_base = "ta_download/_doc/"
|
||||||
valid_status = ["pending", "ignore", "priority"]
|
valid_status = ["pending", "ignore", "ignore-force", "priority"]
|
||||||
|
permission_classes = [AdminOnly]
|
||||||
|
|
||||||
def get(self, request, video_id):
|
def get(self, request, video_id):
|
||||||
# pylint: disable=unused-argument
|
# pylint: disable=unused-argument
|
||||||
|
@ -430,18 +611,20 @@ class DownloadApiView(ApiBaseView):
|
||||||
print(message)
|
print(message)
|
||||||
return Response({"message": message}, status=400)
|
return Response({"message": message}, status=400)
|
||||||
|
|
||||||
|
if item_status == "ignore-force":
|
||||||
|
extrac_dl.delay(video_id, status="ignore")
|
||||||
|
message = f"{video_id}: set status to ignore"
|
||||||
|
return Response(request.data)
|
||||||
|
|
||||||
_, status_code = PendingInteract(video_id).get_item()
|
_, status_code = PendingInteract(video_id).get_item()
|
||||||
if status_code == 404:
|
if status_code == 404:
|
||||||
message = f"{video_id}: item not found {status_code}"
|
message = f"{video_id}: item not found {status_code}"
|
||||||
return Response({"message": message}, status=404)
|
return Response({"message": message}, status=404)
|
||||||
|
|
||||||
print(f"{video_id}: change status to {item_status}")
|
print(f"{video_id}: change status to {item_status}")
|
||||||
|
PendingInteract(video_id, item_status).update_status()
|
||||||
if item_status == "priority":
|
if item_status == "priority":
|
||||||
PendingInteract(youtube_id=video_id).prioritize()
|
download_pending.delay(auto_only=True)
|
||||||
download_pending.delay(from_queue=False)
|
|
||||||
else:
|
|
||||||
PendingInteract(video_id, item_status).update_status()
|
|
||||||
RedisQueue(queue_name="dl_queue").clear_item(video_id)
|
|
||||||
|
|
||||||
return Response(request.data)
|
return Response(request.data)
|
||||||
|
|
||||||
|
@ -464,6 +647,7 @@ class DownloadApiListView(ApiBaseView):
|
||||||
|
|
||||||
search_base = "ta_download/_search/"
|
search_base = "ta_download/_search/"
|
||||||
valid_filter = ["pending", "ignore"]
|
valid_filter = ["pending", "ignore"]
|
||||||
|
permission_classes = [AdminOnly]
|
||||||
|
|
||||||
def get(self, request):
|
def get(self, request):
|
||||||
"""get request"""
|
"""get request"""
|
||||||
|
@ -494,6 +678,7 @@ class DownloadApiListView(ApiBaseView):
|
||||||
def post(request):
|
def post(request):
|
||||||
"""add list of videos to download queue"""
|
"""add list of videos to download queue"""
|
||||||
data = request.data
|
data = request.data
|
||||||
|
auto_start = bool(request.GET.get("autostart"))
|
||||||
try:
|
try:
|
||||||
to_add = data["data"]
|
to_add = data["data"]
|
||||||
except KeyError:
|
except KeyError:
|
||||||
|
@ -510,7 +695,7 @@ class DownloadApiListView(ApiBaseView):
|
||||||
print(message)
|
print(message)
|
||||||
return Response({"message": message}, status=400)
|
return Response({"message": message}, status=400)
|
||||||
|
|
||||||
extrac_dl.delay(youtube_ids)
|
extrac_dl.delay(youtube_ids, auto_start=auto_start)
|
||||||
|
|
||||||
return Response(data)
|
return Response(data)
|
||||||
|
|
||||||
|
@ -537,7 +722,11 @@ class PingView(ApiBaseView):
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def get(request):
|
def get(request):
|
||||||
"""get pong"""
|
"""get pong"""
|
||||||
data = {"response": "pong", "user": request.user.id}
|
data = {
|
||||||
|
"response": "pong",
|
||||||
|
"user": request.user.id,
|
||||||
|
"version": ReleaseVersion().get_local_version(),
|
||||||
|
}
|
||||||
return Response(data)
|
return Response(data)
|
||||||
|
|
||||||
|
|
||||||
|
@ -563,10 +752,12 @@ class LoginApiView(ObtainAuthToken):
|
||||||
|
|
||||||
class SnapshotApiListView(ApiBaseView):
|
class SnapshotApiListView(ApiBaseView):
|
||||||
"""resolves to /api/snapshot/
|
"""resolves to /api/snapshot/
|
||||||
GET: returns snashot config plus list of existing snapshots
|
GET: returns snapshot config plus list of existing snapshots
|
||||||
POST: take snapshot now
|
POST: take snapshot now
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
permission_classes = [AdminOnly]
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def get(request):
|
def get(request):
|
||||||
"""handle get request"""
|
"""handle get request"""
|
||||||
|
@ -591,6 +782,8 @@ class SnapshotApiView(ApiBaseView):
|
||||||
DELETE: delete a snapshot
|
DELETE: delete a snapshot
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
permission_classes = [AdminOnly]
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def get(request, snapshot_id):
|
def get(request, snapshot_id):
|
||||||
"""handle get request"""
|
"""handle get request"""
|
||||||
|
@ -625,11 +818,87 @@ class SnapshotApiView(ApiBaseView):
|
||||||
return Response(response)
|
return Response(response)
|
||||||
|
|
||||||
|
|
||||||
|
class BackupApiListView(ApiBaseView):
|
||||||
|
"""resolves to /api/backup/
|
||||||
|
GET: returns list of available zip backups
|
||||||
|
POST: take zip backup now
|
||||||
|
"""
|
||||||
|
|
||||||
|
permission_classes = [AdminOnly]
|
||||||
|
task_name = "run_backup"
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def get(request):
|
||||||
|
"""handle get request"""
|
||||||
|
# pylint: disable=unused-argument
|
||||||
|
backup_files = ElasticBackup().get_all_backup_files()
|
||||||
|
return Response(backup_files)
|
||||||
|
|
||||||
|
def post(self, request):
|
||||||
|
"""handle post request"""
|
||||||
|
# pylint: disable=unused-argument
|
||||||
|
response = TaskCommand().start(self.task_name)
|
||||||
|
message = {
|
||||||
|
"message": "backup task started",
|
||||||
|
"task_id": response["task_id"],
|
||||||
|
}
|
||||||
|
|
||||||
|
return Response(message)
|
||||||
|
|
||||||
|
|
||||||
|
class BackupApiView(ApiBaseView):
|
||||||
|
"""resolves to /api/backup/<filename>/
|
||||||
|
GET: return a single backup
|
||||||
|
POST: restore backup
|
||||||
|
DELETE: delete backup
|
||||||
|
"""
|
||||||
|
|
||||||
|
permission_classes = [AdminOnly]
|
||||||
|
task_name = "restore_backup"
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def get(request, filename):
|
||||||
|
"""get single backup"""
|
||||||
|
# pylint: disable=unused-argument
|
||||||
|
backup_file = ElasticBackup().build_backup_file_data(filename)
|
||||||
|
if not backup_file:
|
||||||
|
message = {"message": "file not found"}
|
||||||
|
return Response(message, status=404)
|
||||||
|
|
||||||
|
return Response(backup_file)
|
||||||
|
|
||||||
|
def post(self, request, filename):
|
||||||
|
"""restore backup file"""
|
||||||
|
# pylint: disable=unused-argument
|
||||||
|
task = run_restore_backup.delay(filename)
|
||||||
|
message = {
|
||||||
|
"message": "backup restore task started",
|
||||||
|
"filename": filename,
|
||||||
|
"task_id": task.id,
|
||||||
|
}
|
||||||
|
return Response(message)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def delete(request, filename):
|
||||||
|
"""delete backup file"""
|
||||||
|
# pylint: disable=unused-argument
|
||||||
|
|
||||||
|
backup_file = ElasticBackup().delete_file(filename)
|
||||||
|
if not backup_file:
|
||||||
|
message = {"message": "file not found"}
|
||||||
|
return Response(message, status=404)
|
||||||
|
|
||||||
|
message = {"message": f"file {filename} deleted"}
|
||||||
|
return Response(message)
|
||||||
|
|
||||||
|
|
||||||
class TaskListView(ApiBaseView):
|
class TaskListView(ApiBaseView):
|
||||||
"""resolves to /api/task-name/
|
"""resolves to /api/task-name/
|
||||||
GET: return a list of all stored task results
|
GET: return a list of all stored task results
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
permission_classes = [AdminOnly]
|
||||||
|
|
||||||
def get(self, request):
|
def get(self, request):
|
||||||
"""handle get request"""
|
"""handle get request"""
|
||||||
# pylint: disable=unused-argument
|
# pylint: disable=unused-argument
|
||||||
|
@ -644,10 +913,12 @@ class TaskNameListView(ApiBaseView):
|
||||||
POST: start new background process
|
POST: start new background process
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
permission_classes = [AdminOnly]
|
||||||
|
|
||||||
def get(self, request, task_name):
|
def get(self, request, task_name):
|
||||||
"""handle get request"""
|
"""handle get request"""
|
||||||
# pylint: disable=unused-argument
|
# pylint: disable=unused-argument
|
||||||
if task_name not in BaseTask.TASK_CONFIG:
|
if task_name not in TASK_CONFIG:
|
||||||
message = {"message": "invalid task name"}
|
message = {"message": "invalid task name"}
|
||||||
return Response(message, status=404)
|
return Response(message, status=404)
|
||||||
|
|
||||||
|
@ -662,12 +933,12 @@ class TaskNameListView(ApiBaseView):
|
||||||
400 if task can't be started here without argument
|
400 if task can't be started here without argument
|
||||||
"""
|
"""
|
||||||
# pylint: disable=unused-argument
|
# pylint: disable=unused-argument
|
||||||
task_config = BaseTask.TASK_CONFIG.get(task_name)
|
task_config = TASK_CONFIG.get(task_name)
|
||||||
if not task_config:
|
if not task_config:
|
||||||
message = {"message": "invalid task name"}
|
message = {"message": "invalid task name"}
|
||||||
return Response(message, status=404)
|
return Response(message, status=404)
|
||||||
|
|
||||||
if not task_config.get("api-start"):
|
if not task_config.get("api_start"):
|
||||||
message = {"message": "can not start task through this endpoint"}
|
message = {"message": "can not start task through this endpoint"}
|
||||||
return Response(message, status=400)
|
return Response(message, status=400)
|
||||||
|
|
||||||
|
@ -682,6 +953,7 @@ class TaskIDView(ApiBaseView):
|
||||||
"""
|
"""
|
||||||
|
|
||||||
valid_commands = ["stop", "kill"]
|
valid_commands = ["stop", "kill"]
|
||||||
|
permission_classes = [AdminOnly]
|
||||||
|
|
||||||
def get(self, request, task_id):
|
def get(self, request, task_id):
|
||||||
"""handle get request"""
|
"""handle get request"""
|
||||||
|
@ -705,16 +977,16 @@ class TaskIDView(ApiBaseView):
|
||||||
message = {"message": "task id not found"}
|
message = {"message": "task id not found"}
|
||||||
return Response(message, status=404)
|
return Response(message, status=404)
|
||||||
|
|
||||||
task_conf = BaseTask.TASK_CONFIG.get(task_result.get("name"))
|
task_conf = TASK_CONFIG.get(task_result.get("name"))
|
||||||
if command == "stop":
|
if command == "stop":
|
||||||
if not task_conf.get("api-stop"):
|
if not task_conf.get("api_stop"):
|
||||||
message = {"message": "task can not be stopped"}
|
message = {"message": "task can not be stopped"}
|
||||||
return Response(message, status=400)
|
return Response(message, status=400)
|
||||||
|
|
||||||
message_key = self._build_message_key(task_conf, task_id)
|
message_key = self._build_message_key(task_conf, task_id)
|
||||||
TaskCommand().stop(task_id, message_key)
|
TaskCommand().stop(task_id, message_key)
|
||||||
if command == "kill":
|
if command == "kill":
|
||||||
if not task_conf.get("api-stop"):
|
if not task_conf.get("api_stop"):
|
||||||
message = {"message": "task can not be killed"}
|
message = {"message": "task can not be killed"}
|
||||||
return Response(message, status=400)
|
return Response(message, status=400)
|
||||||
|
|
||||||
|
@ -727,12 +999,64 @@ class TaskIDView(ApiBaseView):
|
||||||
return f"message:{task_conf.get('group')}:{task_id.split('-')[0]}"
|
return f"message:{task_conf.get('group')}:{task_id.split('-')[0]}"
|
||||||
|
|
||||||
|
|
||||||
|
class ScheduleView(ApiBaseView):
|
||||||
|
"""resolves to /api/schedule/
|
||||||
|
DEL: delete schedule for task
|
||||||
|
"""
|
||||||
|
|
||||||
|
permission_classes = [AdminOnly]
|
||||||
|
|
||||||
|
def delete(self, request):
|
||||||
|
"""delete schedule by task_name query"""
|
||||||
|
task_name = request.data.get("task_name")
|
||||||
|
try:
|
||||||
|
task = CustomPeriodicTask.objects.get(name=task_name)
|
||||||
|
except CustomPeriodicTask.DoesNotExist:
|
||||||
|
message = {"message": "task_name not found"}
|
||||||
|
return Response(message, status=404)
|
||||||
|
|
||||||
|
_ = task.delete()
|
||||||
|
|
||||||
|
return Response({"success": True})
|
||||||
|
|
||||||
|
|
||||||
|
class ScheduleNotification(ApiBaseView):
|
||||||
|
"""resolves to /api/schedule/notification/
|
||||||
|
GET: get all schedule notifications
|
||||||
|
DEL: delete notification
|
||||||
|
"""
|
||||||
|
|
||||||
|
def get(self, request):
|
||||||
|
"""handle get request"""
|
||||||
|
|
||||||
|
return Response(get_all_notifications())
|
||||||
|
|
||||||
|
def delete(self, request):
|
||||||
|
"""handle delete"""
|
||||||
|
|
||||||
|
task_name = request.data.get("task_name")
|
||||||
|
url = request.data.get("url")
|
||||||
|
|
||||||
|
if not TASK_CONFIG.get(task_name):
|
||||||
|
message = {"message": "task_name not found"}
|
||||||
|
return Response(message, status=404)
|
||||||
|
|
||||||
|
if url:
|
||||||
|
response, status_code = Notifications(task_name).remove_url(url)
|
||||||
|
else:
|
||||||
|
response, status_code = Notifications(task_name).remove_task()
|
||||||
|
|
||||||
|
return Response({"response": response, "status_code": status_code})
|
||||||
|
|
||||||
|
|
||||||
class RefreshView(ApiBaseView):
|
class RefreshView(ApiBaseView):
|
||||||
"""resolves to /api/refresh/
|
"""resolves to /api/refresh/
|
||||||
GET: get refresh progress
|
GET: get refresh progress
|
||||||
POST: start a manual refresh task
|
POST: start a manual refresh task
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
permission_classes = [AdminOnly]
|
||||||
|
|
||||||
def get(self, request):
|
def get(self, request):
|
||||||
"""handle get request"""
|
"""handle get request"""
|
||||||
request_type = request.GET.get("type")
|
request_type = request.GET.get("type")
|
||||||
|
@ -759,6 +1083,42 @@ class RefreshView(ApiBaseView):
|
||||||
return Response(data)
|
return Response(data)
|
||||||
|
|
||||||
|
|
||||||
|
class UserConfigView(ApiBaseView):
|
||||||
|
"""resolves to /api/config/user/
|
||||||
|
GET: return current user config
|
||||||
|
POST: update user config
|
||||||
|
"""
|
||||||
|
|
||||||
|
def get(self, request):
|
||||||
|
"""get config"""
|
||||||
|
user_id = request.user.id
|
||||||
|
response = UserConfig(user_id).get_config()
|
||||||
|
response.update({"user_id": user_id})
|
||||||
|
|
||||||
|
return Response(response)
|
||||||
|
|
||||||
|
def post(self, request):
|
||||||
|
"""update config"""
|
||||||
|
user_id = request.user.id
|
||||||
|
data = request.data
|
||||||
|
|
||||||
|
user_conf = UserConfig(user_id)
|
||||||
|
for key, value in data.items():
|
||||||
|
try:
|
||||||
|
user_conf.set_value(key, value)
|
||||||
|
except ValueError as err:
|
||||||
|
message = {
|
||||||
|
"status": "Bad Request",
|
||||||
|
"message": f"failed updating {key} to '{value}', {err}",
|
||||||
|
}
|
||||||
|
return Response(message, status=400)
|
||||||
|
|
||||||
|
response = user_conf.get_config()
|
||||||
|
response.update({"user_id": user_id})
|
||||||
|
|
||||||
|
return Response(response)
|
||||||
|
|
||||||
|
|
||||||
class CookieView(ApiBaseView):
|
class CookieView(ApiBaseView):
|
||||||
"""resolves to /api/cookie/
|
"""resolves to /api/cookie/
|
||||||
GET: check if cookie is enabled
|
GET: check if cookie is enabled
|
||||||
|
@ -766,6 +1126,8 @@ class CookieView(ApiBaseView):
|
||||||
PUT: import cookie
|
PUT: import cookie
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
permission_classes = [AdminOnly]
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def get(request):
|
def get(request):
|
||||||
"""handle get request"""
|
"""handle get request"""
|
||||||
|
@ -853,6 +1215,8 @@ class TokenView(ApiBaseView):
|
||||||
DELETE: revoke the token
|
DELETE: revoke the token
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
permission_classes = [AdminOnly]
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def delete(request):
|
def delete(request):
|
||||||
print("revoke API token")
|
print("revoke API token")
|
||||||
|
@ -876,3 +1240,94 @@ class NotificationView(ApiBaseView):
|
||||||
query = f"{query}:{filter_by}"
|
query = f"{query}:{filter_by}"
|
||||||
|
|
||||||
return Response(RedisArchivist().list_items(query))
|
return Response(RedisArchivist().list_items(query))
|
||||||
|
|
||||||
|
|
||||||
|
class StatVideoView(ApiBaseView):
|
||||||
|
"""resolves to /api/stats/video/
|
||||||
|
GET: return video stats
|
||||||
|
"""
|
||||||
|
|
||||||
|
def get(self, request):
|
||||||
|
"""get stats"""
|
||||||
|
# pylint: disable=unused-argument
|
||||||
|
|
||||||
|
return Response(Video().process())
|
||||||
|
|
||||||
|
|
||||||
|
class StatChannelView(ApiBaseView):
|
||||||
|
"""resolves to /api/stats/channel/
|
||||||
|
GET: return channel stats
|
||||||
|
"""
|
||||||
|
|
||||||
|
def get(self, request):
|
||||||
|
"""get stats"""
|
||||||
|
# pylint: disable=unused-argument
|
||||||
|
|
||||||
|
return Response(Channel().process())
|
||||||
|
|
||||||
|
|
||||||
|
class StatPlaylistView(ApiBaseView):
|
||||||
|
"""resolves to /api/stats/playlist/
|
||||||
|
GET: return playlist stats
|
||||||
|
"""
|
||||||
|
|
||||||
|
def get(self, request):
|
||||||
|
"""get stats"""
|
||||||
|
# pylint: disable=unused-argument
|
||||||
|
|
||||||
|
return Response(Playlist().process())
|
||||||
|
|
||||||
|
|
||||||
|
class StatDownloadView(ApiBaseView):
|
||||||
|
"""resolves to /api/stats/download/
|
||||||
|
GET: return download stats
|
||||||
|
"""
|
||||||
|
|
||||||
|
def get(self, request):
|
||||||
|
"""get stats"""
|
||||||
|
# pylint: disable=unused-argument
|
||||||
|
|
||||||
|
return Response(Download().process())
|
||||||
|
|
||||||
|
|
||||||
|
class StatWatchProgress(ApiBaseView):
|
||||||
|
"""resolves to /api/stats/watchprogress/
|
||||||
|
GET: return watch/unwatch progress stats
|
||||||
|
"""
|
||||||
|
|
||||||
|
def get(self, request):
|
||||||
|
"""handle get request"""
|
||||||
|
# pylint: disable=unused-argument
|
||||||
|
|
||||||
|
return Response(WatchProgress().process())
|
||||||
|
|
||||||
|
|
||||||
|
class StatDownloadHist(ApiBaseView):
|
||||||
|
"""resolves to /api/stats/downloadhist/
|
||||||
|
GET: return download video count histogram for last days
|
||||||
|
"""
|
||||||
|
|
||||||
|
def get(self, request):
|
||||||
|
"""handle get request"""
|
||||||
|
# pylint: disable=unused-argument
|
||||||
|
|
||||||
|
return Response(DownloadHist().process())
|
||||||
|
|
||||||
|
|
||||||
|
class StatBiggestChannel(ApiBaseView):
|
||||||
|
"""resolves to /api/stats/biggestchannels/
|
||||||
|
GET: return biggest channels
|
||||||
|
param: order
|
||||||
|
"""
|
||||||
|
|
||||||
|
order_choices = ["doc_count", "duration", "media_size"]
|
||||||
|
|
||||||
|
def get(self, request):
|
||||||
|
"""handle get request"""
|
||||||
|
|
||||||
|
order = request.GET.get("order", "doc_count")
|
||||||
|
if order and order not in self.order_choices:
|
||||||
|
message = {"message": f"invalid order parameter {order}"}
|
||||||
|
return Response(message, status=400)
|
||||||
|
|
||||||
|
return Response(BiggestChannel(order).process())
|
||||||
|
|
|
@ -8,6 +8,7 @@ from time import sleep
|
||||||
import requests
|
import requests
|
||||||
from django.core.management.base import BaseCommand, CommandError
|
from django.core.management.base import BaseCommand, CommandError
|
||||||
from home.src.es.connect import ElasticWrap
|
from home.src.es.connect import ElasticWrap
|
||||||
|
from home.src.ta.settings import EnvironmentSettings
|
||||||
from home.src.ta.ta_redis import RedisArchivist
|
from home.src.ta.ta_redis import RedisArchivist
|
||||||
|
|
||||||
TOPIC = """
|
TOPIC = """
|
||||||
|
@ -81,11 +82,16 @@ class Command(BaseCommand):
|
||||||
_, status_code = ElasticWrap("/").get(
|
_, status_code = ElasticWrap("/").get(
|
||||||
timeout=1, print_error=False
|
timeout=1, print_error=False
|
||||||
)
|
)
|
||||||
except requests.exceptions.ConnectionError:
|
except (
|
||||||
|
requests.exceptions.ConnectionError,
|
||||||
|
requests.exceptions.Timeout,
|
||||||
|
):
|
||||||
sleep(5)
|
sleep(5)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if status_code and status_code == 200:
|
if status_code and status_code == 200:
|
||||||
|
path = "_cluster/health?wait_for_status=yellow&timeout=60s"
|
||||||
|
_, _ = ElasticWrap(path).get(timeout=60)
|
||||||
self.stdout.write(
|
self.stdout.write(
|
||||||
self.style.SUCCESS(" ✓ ES connection established")
|
self.style.SUCCESS(" ✓ ES connection established")
|
||||||
)
|
)
|
||||||
|
@ -116,7 +122,7 @@ class Command(BaseCommand):
|
||||||
return
|
return
|
||||||
|
|
||||||
message = (
|
message = (
|
||||||
" 🗙 ES connection failed. "
|
" 🗙 ES version check failed. "
|
||||||
+ f"Expected {self.MIN_MAJOR}.{self.MIN_MINOR} but got {version}"
|
+ f"Expected {self.MIN_MAJOR}.{self.MIN_MINOR} but got {version}"
|
||||||
)
|
)
|
||||||
self.stdout.write(self.style.ERROR(f"{message}"))
|
self.stdout.write(self.style.ERROR(f"{message}"))
|
||||||
|
@ -127,7 +133,19 @@ class Command(BaseCommand):
|
||||||
"""check that path.repo var is set"""
|
"""check that path.repo var is set"""
|
||||||
self.stdout.write("[5] check ES path.repo env var")
|
self.stdout.write("[5] check ES path.repo env var")
|
||||||
response, _ = ElasticWrap("_nodes/_all/settings").get()
|
response, _ = ElasticWrap("_nodes/_all/settings").get()
|
||||||
|
snaphost_roles = [
|
||||||
|
"data",
|
||||||
|
"data_cold",
|
||||||
|
"data_content",
|
||||||
|
"data_frozen",
|
||||||
|
"data_hot",
|
||||||
|
"data_warm",
|
||||||
|
"master",
|
||||||
|
]
|
||||||
for node in response["nodes"].values():
|
for node in response["nodes"].values():
|
||||||
|
if not (set(node["roles"]) & set(snaphost_roles)):
|
||||||
|
continue
|
||||||
|
|
||||||
if node["settings"]["path"].get("repo"):
|
if node["settings"]["path"].get("repo"):
|
||||||
self.stdout.write(
|
self.stdout.write(
|
||||||
self.style.SUCCESS(" ✓ path.repo env var is set")
|
self.style.SUCCESS(" ✓ path.repo env var is set")
|
||||||
|
@ -137,8 +155,9 @@ class Command(BaseCommand):
|
||||||
message = (
|
message = (
|
||||||
" 🗙 path.repo env var not found. "
|
" 🗙 path.repo env var not found. "
|
||||||
+ "set the following env var to the ES container:\n"
|
+ "set the following env var to the ES container:\n"
|
||||||
+ " path.repo=/usr/share/elasticsearch/data/snapshot"
|
+ " path.repo="
|
||||||
|
+ EnvironmentSettings.ES_SNAPSHOT_DIR
|
||||||
)
|
)
|
||||||
self.stdout.write(self.style.ERROR(f"{message}"))
|
self.stdout.write(self.style.ERROR(message))
|
||||||
sleep(60)
|
sleep(60)
|
||||||
raise CommandError(message)
|
raise CommandError(message)
|
||||||
|
|
|
@ -11,6 +11,7 @@ import re
|
||||||
|
|
||||||
from django.core.management.base import BaseCommand, CommandError
|
from django.core.management.base import BaseCommand, CommandError
|
||||||
from home.models import Account
|
from home.models import Account
|
||||||
|
from home.src.ta.settings import EnvironmentSettings
|
||||||
|
|
||||||
LOGO = """
|
LOGO = """
|
||||||
|
|
||||||
|
@ -96,18 +97,14 @@ class Command(BaseCommand):
|
||||||
|
|
||||||
def _elastic_user_overwrite(self):
|
def _elastic_user_overwrite(self):
|
||||||
"""check for ELASTIC_USER overwrite"""
|
"""check for ELASTIC_USER overwrite"""
|
||||||
self.stdout.write("[2] set default ES user")
|
self.stdout.write("[2] check ES user overwrite")
|
||||||
if not os.environ.get("ELASTIC_USER"):
|
env = EnvironmentSettings.ES_USER
|
||||||
os.environ.setdefault("ELASTIC_USER", "elastic")
|
|
||||||
|
|
||||||
env = os.environ.get("ELASTIC_USER")
|
|
||||||
|
|
||||||
self.stdout.write(self.style.SUCCESS(f" ✓ ES user is set to {env}"))
|
self.stdout.write(self.style.SUCCESS(f" ✓ ES user is set to {env}"))
|
||||||
|
|
||||||
def _ta_port_overwrite(self):
|
def _ta_port_overwrite(self):
|
||||||
"""set TA_PORT overwrite for nginx"""
|
"""set TA_PORT overwrite for nginx"""
|
||||||
self.stdout.write("[3] check TA_PORT overwrite")
|
self.stdout.write("[3] check TA_PORT overwrite")
|
||||||
overwrite = os.environ.get("TA_PORT")
|
overwrite = EnvironmentSettings.TA_PORT
|
||||||
if not overwrite:
|
if not overwrite:
|
||||||
self.stdout.write(self.style.SUCCESS(" TA_PORT is not set"))
|
self.stdout.write(self.style.SUCCESS(" TA_PORT is not set"))
|
||||||
return
|
return
|
||||||
|
@ -125,7 +122,7 @@ class Command(BaseCommand):
|
||||||
def _ta_uwsgi_overwrite(self):
|
def _ta_uwsgi_overwrite(self):
|
||||||
"""set TA_UWSGI_PORT overwrite"""
|
"""set TA_UWSGI_PORT overwrite"""
|
||||||
self.stdout.write("[4] check TA_UWSGI_PORT overwrite")
|
self.stdout.write("[4] check TA_UWSGI_PORT overwrite")
|
||||||
overwrite = os.environ.get("TA_UWSGI_PORT")
|
overwrite = EnvironmentSettings.TA_UWSGI_PORT
|
||||||
if not overwrite:
|
if not overwrite:
|
||||||
message = " TA_UWSGI_PORT is not set"
|
message = " TA_UWSGI_PORT is not set"
|
||||||
self.stdout.write(self.style.SUCCESS(message))
|
self.stdout.write(self.style.SUCCESS(message))
|
||||||
|
@ -151,7 +148,7 @@ class Command(BaseCommand):
|
||||||
def _enable_cast_overwrite(self):
|
def _enable_cast_overwrite(self):
|
||||||
"""cast workaround, remove auth for static files in nginx"""
|
"""cast workaround, remove auth for static files in nginx"""
|
||||||
self.stdout.write("[5] check ENABLE_CAST overwrite")
|
self.stdout.write("[5] check ENABLE_CAST overwrite")
|
||||||
overwrite = os.environ.get("ENABLE_CAST")
|
overwrite = EnvironmentSettings.ENABLE_CAST
|
||||||
if not overwrite:
|
if not overwrite:
|
||||||
self.stdout.write(self.style.SUCCESS(" ENABLE_CAST is not set"))
|
self.stdout.write(self.style.SUCCESS(" ENABLE_CAST is not set"))
|
||||||
return
|
return
|
||||||
|
@ -174,8 +171,8 @@ class Command(BaseCommand):
|
||||||
self.stdout.write(self.style.SUCCESS(message))
|
self.stdout.write(self.style.SUCCESS(message))
|
||||||
return
|
return
|
||||||
|
|
||||||
name = os.environ.get("TA_USERNAME")
|
name = EnvironmentSettings.TA_USERNAME
|
||||||
password = os.environ.get("TA_PASSWORD")
|
password = EnvironmentSettings.TA_PASSWORD
|
||||||
Account.objects.create_superuser(name, password)
|
Account.objects.create_superuser(name, password)
|
||||||
message = f" ✓ new superuser with name {name} created"
|
message = f" ✓ new superuser with name {name} created"
|
||||||
self.stdout.write(self.style.SUCCESS(message))
|
self.stdout.write(self.style.SUCCESS(message))
|
||||||
|
|
|
@ -0,0 +1,185 @@
|
||||||
|
"""
|
||||||
|
filepath migration from v0.3.6 to v0.3.7
|
||||||
|
not getting called at startup any more, to run manually if needed:
|
||||||
|
python manage.py ta_migpath
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
|
||||||
|
from django.core.management.base import BaseCommand
|
||||||
|
from home.src.es.connect import ElasticWrap, IndexPaginate
|
||||||
|
from home.src.ta.helper import ignore_filelist
|
||||||
|
from home.src.ta.settings import EnvironmentSettings
|
||||||
|
|
||||||
|
TOPIC = """
|
||||||
|
|
||||||
|
########################
|
||||||
|
# Filesystem Migration #
|
||||||
|
########################
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
class Command(BaseCommand):
|
||||||
|
"""command framework"""
|
||||||
|
|
||||||
|
# pylint: disable=no-member
|
||||||
|
|
||||||
|
def handle(self, *args, **options):
|
||||||
|
"""run commands"""
|
||||||
|
self.stdout.write(TOPIC)
|
||||||
|
|
||||||
|
handler = FolderMigration()
|
||||||
|
to_migrate = handler.get_to_migrate()
|
||||||
|
if not to_migrate:
|
||||||
|
self.stdout.write(
|
||||||
|
self.style.SUCCESS(" no channel migration needed\n")
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
self.stdout.write(self.style.SUCCESS(" migrating channels"))
|
||||||
|
total_channels = handler.create_folders(to_migrate)
|
||||||
|
self.stdout.write(
|
||||||
|
self.style.SUCCESS(f" created {total_channels} channels")
|
||||||
|
)
|
||||||
|
|
||||||
|
self.stdout.write(
|
||||||
|
self.style.SUCCESS(f" migrating {len(to_migrate)} videos")
|
||||||
|
)
|
||||||
|
handler.migrate_videos(to_migrate)
|
||||||
|
self.stdout.write(self.style.SUCCESS(" update videos in index"))
|
||||||
|
handler.send_bulk()
|
||||||
|
|
||||||
|
self.stdout.write(self.style.SUCCESS(" cleanup old folders"))
|
||||||
|
handler.delete_old()
|
||||||
|
|
||||||
|
self.stdout.write(self.style.SUCCESS(" ✓ migration completed\n"))
|
||||||
|
|
||||||
|
|
||||||
|
class FolderMigration:
|
||||||
|
"""migrate video archive folder"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.videos = EnvironmentSettings.MEDIA_DIR
|
||||||
|
self.bulk_list = []
|
||||||
|
|
||||||
|
def get_to_migrate(self):
|
||||||
|
"""get videos to migrate"""
|
||||||
|
script = (
|
||||||
|
"doc['media_url'].value == "
|
||||||
|
+ "doc['channel.channel_id'].value + '/'"
|
||||||
|
+ " + doc['youtube_id'].value + '.mp4'"
|
||||||
|
)
|
||||||
|
data = {
|
||||||
|
"query": {"bool": {"must_not": [{"script": {"script": script}}]}},
|
||||||
|
"_source": [
|
||||||
|
"youtube_id",
|
||||||
|
"media_url",
|
||||||
|
"channel.channel_id",
|
||||||
|
"subtitles",
|
||||||
|
],
|
||||||
|
}
|
||||||
|
response = IndexPaginate("ta_video", data).get_results()
|
||||||
|
|
||||||
|
return response
|
||||||
|
|
||||||
|
def create_folders(self, to_migrate):
|
||||||
|
"""create required channel folders"""
|
||||||
|
host_uid = EnvironmentSettings.HOST_UID
|
||||||
|
host_gid = EnvironmentSettings.HOST_GID
|
||||||
|
all_channel_ids = {i["channel"]["channel_id"] for i in to_migrate}
|
||||||
|
|
||||||
|
for channel_id in all_channel_ids:
|
||||||
|
new_folder = os.path.join(self.videos, channel_id)
|
||||||
|
os.makedirs(new_folder, exist_ok=True)
|
||||||
|
if host_uid and host_gid:
|
||||||
|
os.chown(new_folder, host_uid, host_gid)
|
||||||
|
|
||||||
|
return len(all_channel_ids)
|
||||||
|
|
||||||
|
def migrate_videos(self, to_migrate):
|
||||||
|
"""migrate all videos of channel"""
|
||||||
|
total = len(to_migrate)
|
||||||
|
for idx, video in enumerate(to_migrate):
|
||||||
|
new_media_url = self._move_video_file(video)
|
||||||
|
if not new_media_url:
|
||||||
|
continue
|
||||||
|
|
||||||
|
all_subtitles = self._move_subtitles(video)
|
||||||
|
action = {
|
||||||
|
"update": {"_id": video["youtube_id"], "_index": "ta_video"}
|
||||||
|
}
|
||||||
|
source = {"doc": {"media_url": new_media_url}}
|
||||||
|
if all_subtitles:
|
||||||
|
source["doc"].update({"subtitles": all_subtitles})
|
||||||
|
|
||||||
|
self.bulk_list.append(json.dumps(action))
|
||||||
|
self.bulk_list.append(json.dumps(source))
|
||||||
|
if idx % 1000 == 0:
|
||||||
|
print(f"processing migration [{idx}/{total}]")
|
||||||
|
self.send_bulk()
|
||||||
|
|
||||||
|
def _move_video_file(self, video):
|
||||||
|
"""move video file to new location"""
|
||||||
|
old_path = os.path.join(self.videos, video["media_url"])
|
||||||
|
if not os.path.exists(old_path):
|
||||||
|
print(f"did not find expected video at {old_path}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
new_media_url = os.path.join(
|
||||||
|
video["channel"]["channel_id"], video["youtube_id"] + ".mp4"
|
||||||
|
)
|
||||||
|
new_path = os.path.join(self.videos, new_media_url)
|
||||||
|
os.rename(old_path, new_path)
|
||||||
|
|
||||||
|
return new_media_url
|
||||||
|
|
||||||
|
def _move_subtitles(self, video):
|
||||||
|
"""move subtitle files to new location"""
|
||||||
|
all_subtitles = video.get("subtitles")
|
||||||
|
if not all_subtitles:
|
||||||
|
return False
|
||||||
|
|
||||||
|
for subtitle in all_subtitles:
|
||||||
|
old_path = os.path.join(self.videos, subtitle["media_url"])
|
||||||
|
if not os.path.exists(old_path):
|
||||||
|
print(f"did not find expected subtitle at {old_path}")
|
||||||
|
continue
|
||||||
|
|
||||||
|
new_media_url = os.path.join(
|
||||||
|
video["channel"]["channel_id"],
|
||||||
|
f"{video.get('youtube_id')}.{subtitle.get('lang')}.vtt",
|
||||||
|
)
|
||||||
|
new_path = os.path.join(self.videos, new_media_url)
|
||||||
|
os.rename(old_path, new_path)
|
||||||
|
subtitle["media_url"] = new_media_url
|
||||||
|
|
||||||
|
return all_subtitles
|
||||||
|
|
||||||
|
def send_bulk(self):
|
||||||
|
"""send bulk request to update index with new urls"""
|
||||||
|
if not self.bulk_list:
|
||||||
|
print("nothing to update")
|
||||||
|
return
|
||||||
|
|
||||||
|
self.bulk_list.append("\n")
|
||||||
|
path = "_bulk?refresh=true"
|
||||||
|
data = "\n".join(self.bulk_list)
|
||||||
|
response, status = ElasticWrap(path).post(data=data, ndjson=True)
|
||||||
|
if not status == 200:
|
||||||
|
print(response)
|
||||||
|
|
||||||
|
self.bulk_list = []
|
||||||
|
|
||||||
|
def delete_old(self):
|
||||||
|
"""delete old empty folders"""
|
||||||
|
all_folders = ignore_filelist(os.listdir(self.videos))
|
||||||
|
for folder in all_folders:
|
||||||
|
folder_path = os.path.join(self.videos, folder)
|
||||||
|
if not os.path.isdir(folder_path):
|
||||||
|
continue
|
||||||
|
|
||||||
|
if not ignore_filelist(os.listdir(folder_path)):
|
||||||
|
shutil.rmtree(folder_path)
|
|
@ -5,15 +5,23 @@ Functionality:
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
from random import randint
|
||||||
from time import sleep
|
from time import sleep
|
||||||
|
|
||||||
|
from django.conf import settings
|
||||||
from django.core.management.base import BaseCommand, CommandError
|
from django.core.management.base import BaseCommand, CommandError
|
||||||
|
from django_celery_beat.models import CrontabSchedule
|
||||||
|
from home.models import CustomPeriodicTask
|
||||||
from home.src.es.connect import ElasticWrap
|
from home.src.es.connect import ElasticWrap
|
||||||
from home.src.es.index_setup import ElasitIndexWrap
|
from home.src.es.index_setup import ElasitIndexWrap
|
||||||
from home.src.es.snapshot import ElasticSnapshot
|
from home.src.es.snapshot import ElasticSnapshot
|
||||||
from home.src.ta.config import AppConfig, ReleaseVersion
|
from home.src.ta.config import AppConfig, ReleaseVersion
|
||||||
|
from home.src.ta.config_schedule import ScheduleBuilder
|
||||||
from home.src.ta.helper import clear_dl_cache
|
from home.src.ta.helper import clear_dl_cache
|
||||||
|
from home.src.ta.notify import Notifications
|
||||||
|
from home.src.ta.settings import EnvironmentSettings
|
||||||
from home.src.ta.ta_redis import RedisArchivist
|
from home.src.ta.ta_redis import RedisArchivist
|
||||||
|
from home.src.ta.task_config import TASK_CONFIG
|
||||||
from home.src.ta.task_manager import TaskManager
|
from home.src.ta.task_manager import TaskManager
|
||||||
|
|
||||||
TOPIC = """
|
TOPIC = """
|
||||||
|
@ -35,13 +43,15 @@ class Command(BaseCommand):
|
||||||
self.stdout.write(TOPIC)
|
self.stdout.write(TOPIC)
|
||||||
self._sync_redis_state()
|
self._sync_redis_state()
|
||||||
self._make_folders()
|
self._make_folders()
|
||||||
self._release_locks()
|
self._clear_redis_keys()
|
||||||
self._clear_tasks()
|
self._clear_tasks()
|
||||||
self._clear_dl_cache()
|
self._clear_dl_cache()
|
||||||
self._version_check()
|
self._version_check()
|
||||||
self._mig_index_setup()
|
self._mig_index_setup()
|
||||||
self._mig_snapshot_check()
|
self._mig_snapshot_check()
|
||||||
self._mig_set_vid_type()
|
self._mig_schedule_store()
|
||||||
|
self._mig_custom_playlist()
|
||||||
|
self._create_default_schedules()
|
||||||
|
|
||||||
def _sync_redis_state(self):
|
def _sync_redis_state(self):
|
||||||
"""make sure redis gets new config.json values"""
|
"""make sure redis gets new config.json values"""
|
||||||
|
@ -65,17 +75,17 @@ class Command(BaseCommand):
|
||||||
"playlists",
|
"playlists",
|
||||||
"videos",
|
"videos",
|
||||||
]
|
]
|
||||||
cache_dir = AppConfig().config["application"]["cache_dir"]
|
cache_dir = EnvironmentSettings.CACHE_DIR
|
||||||
for folder in folders:
|
for folder in folders:
|
||||||
folder_path = os.path.join(cache_dir, folder)
|
folder_path = os.path.join(cache_dir, folder)
|
||||||
os.makedirs(folder_path, exist_ok=True)
|
os.makedirs(folder_path, exist_ok=True)
|
||||||
|
|
||||||
self.stdout.write(self.style.SUCCESS(" ✓ expected folders created"))
|
self.stdout.write(self.style.SUCCESS(" ✓ expected folders created"))
|
||||||
|
|
||||||
def _release_locks(self):
|
def _clear_redis_keys(self):
|
||||||
"""make sure there are no leftover locks set in redis"""
|
"""make sure there are no leftover locks or keys set in redis"""
|
||||||
self.stdout.write("[3] clear leftover locks in redis")
|
self.stdout.write("[3] clear leftover keys in redis")
|
||||||
all_locks = [
|
all_keys = [
|
||||||
"dl_queue_id",
|
"dl_queue_id",
|
||||||
"dl_queue",
|
"dl_queue",
|
||||||
"downloading",
|
"downloading",
|
||||||
|
@ -84,19 +94,22 @@ class Command(BaseCommand):
|
||||||
"rescan",
|
"rescan",
|
||||||
"run_backup",
|
"run_backup",
|
||||||
"startup_check",
|
"startup_check",
|
||||||
|
"reindex:ta_video",
|
||||||
|
"reindex:ta_channel",
|
||||||
|
"reindex:ta_playlist",
|
||||||
]
|
]
|
||||||
|
|
||||||
redis_con = RedisArchivist()
|
redis_con = RedisArchivist()
|
||||||
has_changed = False
|
has_changed = False
|
||||||
for lock in all_locks:
|
for key in all_keys:
|
||||||
if redis_con.del_message(lock):
|
if redis_con.del_message(key):
|
||||||
self.stdout.write(
|
self.stdout.write(
|
||||||
self.style.SUCCESS(f" ✓ cleared lock {lock}")
|
self.style.SUCCESS(f" ✓ cleared key {key}")
|
||||||
)
|
)
|
||||||
has_changed = True
|
has_changed = True
|
||||||
|
|
||||||
if not has_changed:
|
if not has_changed:
|
||||||
self.stdout.write(self.style.SUCCESS(" no locks found"))
|
self.stdout.write(self.style.SUCCESS(" no keys found"))
|
||||||
|
|
||||||
def _clear_tasks(self):
|
def _clear_tasks(self):
|
||||||
"""clear tasks and messages"""
|
"""clear tasks and messages"""
|
||||||
|
@ -115,8 +128,7 @@ class Command(BaseCommand):
|
||||||
def _clear_dl_cache(self):
|
def _clear_dl_cache(self):
|
||||||
"""clear leftover files from dl cache"""
|
"""clear leftover files from dl cache"""
|
||||||
self.stdout.write("[5] clear leftover files from dl cache")
|
self.stdout.write("[5] clear leftover files from dl cache")
|
||||||
config = AppConfig().config
|
leftover_files = clear_dl_cache(EnvironmentSettings.CACHE_DIR)
|
||||||
leftover_files = clear_dl_cache(config)
|
|
||||||
if leftover_files:
|
if leftover_files:
|
||||||
self.stdout.write(
|
self.stdout.write(
|
||||||
self.style.SUCCESS(f" ✓ cleared {leftover_files} files")
|
self.style.SUCCESS(f" ✓ cleared {leftover_files} files")
|
||||||
|
@ -145,51 +157,212 @@ class Command(BaseCommand):
|
||||||
self.stdout.write("[MIGRATION] setup snapshots")
|
self.stdout.write("[MIGRATION] setup snapshots")
|
||||||
ElasticSnapshot().setup()
|
ElasticSnapshot().setup()
|
||||||
|
|
||||||
def _mig_set_vid_type(self):
|
def _mig_schedule_store(self):
|
||||||
"""migration: update 0.3.0 to 0.3.1 set vid_type default"""
|
"""
|
||||||
self.stdout.write("[MIGRATION] set default vid_type")
|
update from 0.4.7 to 0.4.8
|
||||||
index_list = ["ta_video", "ta_download"]
|
migrate schedule task store to CustomCronSchedule
|
||||||
|
"""
|
||||||
|
self.stdout.write("[MIGRATION] migrate schedule store")
|
||||||
|
config = AppConfig().config
|
||||||
|
current_schedules = config.get("scheduler")
|
||||||
|
if not current_schedules:
|
||||||
|
self.stdout.write(
|
||||||
|
self.style.SUCCESS(" no schedules to migrate")
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
self._mig_update_subscribed(current_schedules)
|
||||||
|
self._mig_download_pending(current_schedules)
|
||||||
|
self._mig_check_reindex(current_schedules)
|
||||||
|
self._mig_thumbnail_check(current_schedules)
|
||||||
|
self._mig_run_backup(current_schedules)
|
||||||
|
self._mig_version_check()
|
||||||
|
|
||||||
|
del config["scheduler"]
|
||||||
|
RedisArchivist().set_message("config", config, save=True)
|
||||||
|
|
||||||
|
def _mig_update_subscribed(self, current_schedules):
|
||||||
|
"""create update_subscribed schedule"""
|
||||||
|
task_name = "update_subscribed"
|
||||||
|
update_subscribed_schedule = current_schedules.get(task_name)
|
||||||
|
if update_subscribed_schedule:
|
||||||
|
self._create_task(task_name, update_subscribed_schedule)
|
||||||
|
|
||||||
|
self._create_notifications(task_name, current_schedules)
|
||||||
|
|
||||||
|
def _mig_download_pending(self, current_schedules):
|
||||||
|
"""create download_pending schedule"""
|
||||||
|
task_name = "download_pending"
|
||||||
|
download_pending_schedule = current_schedules.get(task_name)
|
||||||
|
if download_pending_schedule:
|
||||||
|
self._create_task(task_name, download_pending_schedule)
|
||||||
|
|
||||||
|
self._create_notifications(task_name, current_schedules)
|
||||||
|
|
||||||
|
def _mig_check_reindex(self, current_schedules):
|
||||||
|
"""create check_reindex schedule"""
|
||||||
|
task_name = "check_reindex"
|
||||||
|
check_reindex_schedule = current_schedules.get(task_name)
|
||||||
|
if check_reindex_schedule:
|
||||||
|
task_config = {}
|
||||||
|
days = current_schedules.get("check_reindex_days")
|
||||||
|
if days:
|
||||||
|
task_config.update({"days": days})
|
||||||
|
|
||||||
|
self._create_task(
|
||||||
|
task_name,
|
||||||
|
check_reindex_schedule,
|
||||||
|
task_config=task_config,
|
||||||
|
)
|
||||||
|
|
||||||
|
self._create_notifications(task_name, current_schedules)
|
||||||
|
|
||||||
|
def _mig_thumbnail_check(self, current_schedules):
|
||||||
|
"""create thumbnail_check schedule"""
|
||||||
|
thumbnail_check_schedule = current_schedules.get("thumbnail_check")
|
||||||
|
if thumbnail_check_schedule:
|
||||||
|
self._create_task("thumbnail_check", thumbnail_check_schedule)
|
||||||
|
|
||||||
|
def _mig_run_backup(self, current_schedules):
|
||||||
|
"""create run_backup schedule"""
|
||||||
|
run_backup_schedule = current_schedules.get("run_backup")
|
||||||
|
if run_backup_schedule:
|
||||||
|
task_config = False
|
||||||
|
rotate = current_schedules.get("run_backup_rotate")
|
||||||
|
if rotate:
|
||||||
|
task_config = {"rotate": rotate}
|
||||||
|
|
||||||
|
self._create_task(
|
||||||
|
"run_backup", run_backup_schedule, task_config=task_config
|
||||||
|
)
|
||||||
|
|
||||||
|
def _mig_version_check(self):
|
||||||
|
"""create version_check schedule"""
|
||||||
|
version_check_schedule = {
|
||||||
|
"minute": randint(0, 59),
|
||||||
|
"hour": randint(0, 23),
|
||||||
|
"day_of_week": "*",
|
||||||
|
}
|
||||||
|
self._create_task("version_check", version_check_schedule)
|
||||||
|
|
||||||
|
def _create_task(self, task_name, schedule, task_config=False):
|
||||||
|
"""create task"""
|
||||||
|
description = TASK_CONFIG[task_name].get("title")
|
||||||
|
schedule, _ = CrontabSchedule.objects.get_or_create(**schedule)
|
||||||
|
schedule.timezone = settings.TIME_ZONE
|
||||||
|
schedule.save()
|
||||||
|
|
||||||
|
task, _ = CustomPeriodicTask.objects.get_or_create(
|
||||||
|
crontab=schedule,
|
||||||
|
name=task_name,
|
||||||
|
description=description,
|
||||||
|
task=task_name,
|
||||||
|
)
|
||||||
|
if task_config:
|
||||||
|
task.task_config = task_config
|
||||||
|
task.save()
|
||||||
|
|
||||||
|
self.stdout.write(
|
||||||
|
self.style.SUCCESS(f" ✓ new task created: '{task}'")
|
||||||
|
)
|
||||||
|
|
||||||
|
def _create_notifications(self, task_name, current_schedules):
|
||||||
|
"""migrate notifications of task"""
|
||||||
|
notifications = current_schedules.get(f"{task_name}_notify")
|
||||||
|
if not notifications:
|
||||||
|
return
|
||||||
|
|
||||||
|
urls = [i.strip() for i in notifications.split()]
|
||||||
|
if not urls:
|
||||||
|
return
|
||||||
|
|
||||||
|
self.stdout.write(
|
||||||
|
self.style.SUCCESS(f" ✓ migrate notifications: '{urls}'")
|
||||||
|
)
|
||||||
|
handler = Notifications(task_name)
|
||||||
|
for url in urls:
|
||||||
|
handler.add_url(url)
|
||||||
|
|
||||||
|
def _mig_custom_playlist(self):
|
||||||
|
"""add playlist_type for migration from v0.4.6 to v0.4.7"""
|
||||||
|
self.stdout.write("[MIGRATION] custom playlist")
|
||||||
data = {
|
data = {
|
||||||
"query": {
|
"query": {
|
||||||
"bool": {
|
"bool": {"must_not": [{"exists": {"field": "playlist_type"}}]}
|
||||||
"should": [
|
|
||||||
{
|
|
||||||
"bool": {
|
|
||||||
"must_not": [{"exists": {"field": "vid_type"}}]
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{"term": {"vid_type": {"value": "unknown"}}},
|
|
||||||
]
|
|
||||||
}
|
|
||||||
},
|
},
|
||||||
"script": {"source": "ctx._source['vid_type'] = 'videos'"},
|
"script": {"source": "ctx._source['playlist_type'] = 'regular'"},
|
||||||
}
|
}
|
||||||
|
path = "ta_playlist/_update_by_query"
|
||||||
for index_name in index_list:
|
response, status_code = ElasticWrap(path).post(data=data)
|
||||||
path = f"{index_name}/_update_by_query"
|
if status_code == 200:
|
||||||
response, status_code = ElasticWrap(path).post(data=data)
|
updated = response.get("updated", 0)
|
||||||
if status_code == 503:
|
if updated:
|
||||||
message = f" 🗙 {index_name} retry failed migration."
|
|
||||||
self.stdout.write(self.style.ERROR(message))
|
|
||||||
sleep(10)
|
|
||||||
response, status_code = ElasticWrap(path).post(data=data)
|
|
||||||
|
|
||||||
if status_code == 200:
|
|
||||||
updated = response.get("updated", 0)
|
|
||||||
if not updated:
|
|
||||||
self.stdout.write(
|
|
||||||
f" no videos needed updating in {index_name}"
|
|
||||||
)
|
|
||||||
continue
|
|
||||||
|
|
||||||
self.stdout.write(
|
self.stdout.write(
|
||||||
self.style.SUCCESS(
|
self.style.SUCCESS(
|
||||||
f" ✓ {updated} videos updated in {index_name}"
|
f" ✓ {updated} playlist_type updated in ta_playlist"
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
message = f" 🗙 {index_name} vid_type update failed"
|
self.stdout.write(
|
||||||
self.stdout.write(self.style.ERROR(message))
|
self.style.SUCCESS(
|
||||||
self.stdout.write(response)
|
" no playlist_type needed updating in ta_playlist"
|
||||||
sleep(60)
|
)
|
||||||
raise CommandError(message)
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
message = " 🗙 ta_playlist playlist_type update failed"
|
||||||
|
self.stdout.write(self.style.ERROR(message))
|
||||||
|
self.stdout.write(response)
|
||||||
|
sleep(60)
|
||||||
|
raise CommandError(message)
|
||||||
|
|
||||||
|
def _create_default_schedules(self) -> None:
|
||||||
|
"""
|
||||||
|
create default schedules for new installations
|
||||||
|
needs to be called after _mig_schedule_store
|
||||||
|
"""
|
||||||
|
self.stdout.write("[7] create initial schedules")
|
||||||
|
init_has_run = CustomPeriodicTask.objects.filter(
|
||||||
|
name="version_check"
|
||||||
|
).exists()
|
||||||
|
|
||||||
|
if init_has_run:
|
||||||
|
self.stdout.write(
|
||||||
|
self.style.SUCCESS(
|
||||||
|
" schedule init already done, skipping..."
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
builder = ScheduleBuilder()
|
||||||
|
check_reindex = builder.get_set_task(
|
||||||
|
"check_reindex", schedule=builder.SCHEDULES["check_reindex"]
|
||||||
|
)
|
||||||
|
check_reindex.task_config.update({"days": 90})
|
||||||
|
check_reindex.save()
|
||||||
|
self.stdout.write(
|
||||||
|
self.style.SUCCESS(
|
||||||
|
f" ✓ created new default schedule: {check_reindex}"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
thumbnail_check = builder.get_set_task(
|
||||||
|
"thumbnail_check", schedule=builder.SCHEDULES["thumbnail_check"]
|
||||||
|
)
|
||||||
|
self.stdout.write(
|
||||||
|
self.style.SUCCESS(
|
||||||
|
f" ✓ created new default schedule: {thumbnail_check}"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
daily_random = f"{randint(0, 59)} {randint(0, 23)} *"
|
||||||
|
version_check = builder.get_set_task(
|
||||||
|
"version_check", schedule=daily_random
|
||||||
|
)
|
||||||
|
self.stdout.write(
|
||||||
|
self.style.SUCCESS(
|
||||||
|
f" ✓ created new default schedule: {version_check}"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
self.stdout.write(
|
||||||
|
self.style.SUCCESS(" ✓ all default schedules created")
|
||||||
|
)
|
||||||
|
|
|
@ -17,8 +17,8 @@ from pathlib import Path
|
||||||
import ldap
|
import ldap
|
||||||
from corsheaders.defaults import default_headers
|
from corsheaders.defaults import default_headers
|
||||||
from django_auth_ldap.config import LDAPSearch
|
from django_auth_ldap.config import LDAPSearch
|
||||||
from home.src.ta.config import AppConfig
|
|
||||||
from home.src.ta.helper import ta_host_parser
|
from home.src.ta.helper import ta_host_parser
|
||||||
|
from home.src.ta.settings import EnvironmentSettings
|
||||||
|
|
||||||
# Build paths inside the project like this: BASE_DIR / 'subdir'.
|
# Build paths inside the project like this: BASE_DIR / 'subdir'.
|
||||||
BASE_DIR = Path(__file__).resolve().parent.parent
|
BASE_DIR = Path(__file__).resolve().parent.parent
|
||||||
|
@ -27,7 +27,7 @@ BASE_DIR = Path(__file__).resolve().parent.parent
|
||||||
# Quick-start development settings - unsuitable for production
|
# Quick-start development settings - unsuitable for production
|
||||||
# See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/
|
# See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/
|
||||||
|
|
||||||
PW_HASH = hashlib.sha256(environ["TA_PASSWORD"].encode())
|
PW_HASH = hashlib.sha256(EnvironmentSettings.TA_PASSWORD.encode())
|
||||||
SECRET_KEY = PW_HASH.hexdigest()
|
SECRET_KEY = PW_HASH.hexdigest()
|
||||||
|
|
||||||
# SECURITY WARNING: don't run with debug turned on in production!
|
# SECURITY WARNING: don't run with debug turned on in production!
|
||||||
|
@ -38,6 +38,7 @@ ALLOWED_HOSTS, CSRF_TRUSTED_ORIGINS = ta_host_parser(environ["TA_HOST"])
|
||||||
# Application definition
|
# Application definition
|
||||||
|
|
||||||
INSTALLED_APPS = [
|
INSTALLED_APPS = [
|
||||||
|
"django_celery_beat",
|
||||||
"home.apps.HomeConfig",
|
"home.apps.HomeConfig",
|
||||||
"django.contrib.admin",
|
"django.contrib.admin",
|
||||||
"django.contrib.auth",
|
"django.contrib.auth",
|
||||||
|
@ -64,6 +65,7 @@ MIDDLEWARE = [
|
||||||
"django.contrib.auth.middleware.AuthenticationMiddleware",
|
"django.contrib.auth.middleware.AuthenticationMiddleware",
|
||||||
"django.contrib.messages.middleware.MessageMiddleware",
|
"django.contrib.messages.middleware.MessageMiddleware",
|
||||||
"django.middleware.clickjacking.XFrameOptionsMiddleware",
|
"django.middleware.clickjacking.XFrameOptionsMiddleware",
|
||||||
|
"home.src.ta.health.HealthCheckMiddleware",
|
||||||
]
|
]
|
||||||
|
|
||||||
ROOT_URLCONF = "config.urls"
|
ROOT_URLCONF = "config.urls"
|
||||||
|
@ -174,13 +176,12 @@ if bool(environ.get("TA_LDAP")):
|
||||||
ldap.OPT_X_TLS_REQUIRE_CERT: ldap.OPT_X_TLS_NEVER,
|
ldap.OPT_X_TLS_REQUIRE_CERT: ldap.OPT_X_TLS_NEVER,
|
||||||
}
|
}
|
||||||
|
|
||||||
global AUTHENTICATION_BACKENDS
|
|
||||||
AUTHENTICATION_BACKENDS = ("django_auth_ldap.backend.LDAPBackend",)
|
AUTHENTICATION_BACKENDS = ("django_auth_ldap.backend.LDAPBackend",)
|
||||||
|
|
||||||
# Database
|
# Database
|
||||||
# https://docs.djangoproject.com/en/3.2/ref/settings/#databases
|
# https://docs.djangoproject.com/en/3.2/ref/settings/#databases
|
||||||
|
|
||||||
CACHE_DIR = AppConfig().config["application"]["cache_dir"]
|
CACHE_DIR = EnvironmentSettings.CACHE_DIR
|
||||||
DB_PATH = path.join(CACHE_DIR, "db.sqlite3")
|
DB_PATH = path.join(CACHE_DIR, "db.sqlite3")
|
||||||
DATABASES = {
|
DATABASES = {
|
||||||
"default": {
|
"default": {
|
||||||
|
@ -210,12 +211,25 @@ AUTH_PASSWORD_VALIDATORS = [
|
||||||
|
|
||||||
AUTH_USER_MODEL = "home.Account"
|
AUTH_USER_MODEL = "home.Account"
|
||||||
|
|
||||||
|
# Forward-auth authentication
|
||||||
|
if bool(environ.get("TA_ENABLE_AUTH_PROXY")):
|
||||||
|
TA_AUTH_PROXY_USERNAME_HEADER = (
|
||||||
|
environ.get("TA_AUTH_PROXY_USERNAME_HEADER") or "HTTP_REMOTE_USER"
|
||||||
|
)
|
||||||
|
TA_AUTH_PROXY_LOGOUT_URL = environ.get("TA_AUTH_PROXY_LOGOUT_URL")
|
||||||
|
|
||||||
|
MIDDLEWARE.append("home.src.ta.auth.HttpRemoteUserMiddleware")
|
||||||
|
|
||||||
|
AUTHENTICATION_BACKENDS = (
|
||||||
|
"django.contrib.auth.backends.RemoteUserBackend",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
# Internationalization
|
# Internationalization
|
||||||
# https://docs.djangoproject.com/en/3.2/topics/i18n/
|
# https://docs.djangoproject.com/en/3.2/topics/i18n/
|
||||||
|
|
||||||
LANGUAGE_CODE = "en-us"
|
LANGUAGE_CODE = "en-us"
|
||||||
TIME_ZONE = environ.get("TZ") or "UTC"
|
TIME_ZONE = EnvironmentSettings.TZ
|
||||||
USE_I18N = True
|
USE_I18N = True
|
||||||
USE_L10N = True
|
USE_L10N = True
|
||||||
USE_TZ = True
|
USE_TZ = True
|
||||||
|
@ -256,4 +270,4 @@ CORS_ALLOW_HEADERS = list(default_headers) + [
|
||||||
|
|
||||||
# TA application settings
|
# TA application settings
|
||||||
TA_UPSTREAM = "https://github.com/tubearchivist/tubearchivist"
|
TA_UPSTREAM = "https://github.com/tubearchivist/tubearchivist"
|
||||||
TA_VERSION = "v0.3.5"
|
TA_VERSION = "v0.4.8-unstable"
|
||||||
|
|
|
@ -13,6 +13,7 @@ Including another URLconf
|
||||||
1. Import the include() function: from django.urls import include, path
|
1. Import the include() function: from django.urls import include, path
|
||||||
2. Add a URL to urlpatterns: path('blog/', include('blog.urls'))
|
2. Add a URL to urlpatterns: path('blog/', include('blog.urls'))
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from django.contrib import admin
|
from django.contrib import admin
|
||||||
from django.urls import include, path
|
from django.urls import include, path
|
||||||
|
|
||||||
|
|
|
@ -1,5 +1,7 @@
|
||||||
""" handle celery startup """
|
"""start celery app"""
|
||||||
|
|
||||||
from .tasks import app as celery_app
|
from __future__ import absolute_import, unicode_literals
|
||||||
|
|
||||||
|
from home.celery import app as celery_app
|
||||||
|
|
||||||
__all__ = ("celery_app",)
|
__all__ = ("celery_app",)
|
||||||
|
|
|
@ -2,6 +2,7 @@
|
||||||
|
|
||||||
from django.contrib import admin
|
from django.contrib import admin
|
||||||
from django.contrib.auth.admin import UserAdmin as BaseUserAdmin
|
from django.contrib.auth.admin import UserAdmin as BaseUserAdmin
|
||||||
|
from django_celery_beat import models as BeatModels
|
||||||
|
|
||||||
from .models import Account
|
from .models import Account
|
||||||
|
|
||||||
|
@ -34,3 +35,12 @@ class HomeAdmin(BaseUserAdmin):
|
||||||
|
|
||||||
|
|
||||||
admin.site.register(Account, HomeAdmin)
|
admin.site.register(Account, HomeAdmin)
|
||||||
|
admin.site.unregister(
|
||||||
|
[
|
||||||
|
BeatModels.ClockedSchedule,
|
||||||
|
BeatModels.CrontabSchedule,
|
||||||
|
BeatModels.IntervalSchedule,
|
||||||
|
BeatModels.PeriodicTask,
|
||||||
|
BeatModels.SolarSchedule,
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
|
@ -0,0 +1,24 @@
|
||||||
|
"""initiate celery"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
|
from celery import Celery
|
||||||
|
from home.src.ta.config import AppConfig
|
||||||
|
from home.src.ta.settings import EnvironmentSettings
|
||||||
|
|
||||||
|
CONFIG = AppConfig().config
|
||||||
|
REDIS_HOST = EnvironmentSettings.REDIS_HOST
|
||||||
|
REDIS_PORT = EnvironmentSettings.REDIS_PORT
|
||||||
|
|
||||||
|
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
|
||||||
|
app = Celery(
|
||||||
|
"tasks",
|
||||||
|
broker=f"redis://{REDIS_HOST}:{REDIS_PORT}",
|
||||||
|
backend=f"redis://{REDIS_HOST}:{REDIS_PORT}",
|
||||||
|
result_extended=True,
|
||||||
|
)
|
||||||
|
app.config_from_object(
|
||||||
|
"django.conf:settings", namespace=EnvironmentSettings.REDIS_NAME_SPACE
|
||||||
|
)
|
||||||
|
app.autodiscover_tasks()
|
||||||
|
app.conf.timezone = EnvironmentSettings.TZ
|
|
@ -1,29 +1,16 @@
|
||||||
{
|
{
|
||||||
"archive": {
|
|
||||||
"sort_by": "published",
|
|
||||||
"sort_order": "desc",
|
|
||||||
"page_size": 12
|
|
||||||
},
|
|
||||||
"default_view": {
|
|
||||||
"home": "grid",
|
|
||||||
"channel": "list",
|
|
||||||
"downloads": "list",
|
|
||||||
"playlist": "grid",
|
|
||||||
"grid_items": 3
|
|
||||||
},
|
|
||||||
"subscriptions": {
|
"subscriptions": {
|
||||||
"auto_search": false,
|
|
||||||
"auto_download": false,
|
|
||||||
"channel_size": 50,
|
"channel_size": 50,
|
||||||
"live_channel_size": 50,
|
"live_channel_size": 50,
|
||||||
"shorts_channel_size": 50
|
"shorts_channel_size": 50,
|
||||||
|
"auto_start": false
|
||||||
},
|
},
|
||||||
"downloads": {
|
"downloads": {
|
||||||
"limit_count": false,
|
|
||||||
"limit_speed": false,
|
"limit_speed": false,
|
||||||
"sleep_interval": 3,
|
"sleep_interval": 3,
|
||||||
"autodelete_days": false,
|
"autodelete_days": false,
|
||||||
"format": false,
|
"format": false,
|
||||||
|
"format_sort": false,
|
||||||
"add_metadata": false,
|
"add_metadata": false,
|
||||||
"add_thumbnail": false,
|
"add_thumbnail": false,
|
||||||
"subtitle": false,
|
"subtitle": false,
|
||||||
|
@ -33,25 +20,11 @@
|
||||||
"comment_sort": "top",
|
"comment_sort": "top",
|
||||||
"cookie_import": false,
|
"cookie_import": false,
|
||||||
"throttledratelimit": false,
|
"throttledratelimit": false,
|
||||||
|
"extractor_lang": false,
|
||||||
"integrate_ryd": false,
|
"integrate_ryd": false,
|
||||||
"integrate_sponsorblock": false
|
"integrate_sponsorblock": false
|
||||||
},
|
},
|
||||||
"application": {
|
"application": {
|
||||||
"app_root": "/app",
|
|
||||||
"cache_dir": "/cache",
|
|
||||||
"videos": "/youtube",
|
|
||||||
"colors": "dark",
|
|
||||||
"enable_cast": false,
|
|
||||||
"enable_snapshot": true
|
"enable_snapshot": true
|
||||||
},
|
|
||||||
"scheduler": {
|
|
||||||
"update_subscribed": false,
|
|
||||||
"download_pending": false,
|
|
||||||
"check_reindex": {"minute": "0", "hour": "12", "day_of_week": "*"},
|
|
||||||
"check_reindex_days": 90,
|
|
||||||
"thumbnail_check": {"minute": "0", "hour": "17", "day_of_week": "*"},
|
|
||||||
"run_backup": false,
|
|
||||||
"run_backup_rotate": 5,
|
|
||||||
"version_check": "rand-d"
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,23 @@
|
||||||
|
# Generated by Django 4.2.7 on 2023-12-05 13:47
|
||||||
|
|
||||||
|
from django.db import migrations, models
|
||||||
|
import django.db.models.deletion
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('django_celery_beat', '0018_improve_crontab_helptext'),
|
||||||
|
('home', '0001_initial'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.CreateModel(
|
||||||
|
name='CustomPeriodicTask',
|
||||||
|
fields=[
|
||||||
|
('periodictask_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='django_celery_beat.periodictask')),
|
||||||
|
('task_config', models.JSONField(default=dict)),
|
||||||
|
],
|
||||||
|
bases=('django_celery_beat.periodictask',),
|
||||||
|
),
|
||||||
|
]
|
|
@ -1,10 +1,12 @@
|
||||||
"""custom models"""
|
"""custom models"""
|
||||||
|
|
||||||
from django.contrib.auth.models import (
|
from django.contrib.auth.models import (
|
||||||
AbstractBaseUser,
|
AbstractBaseUser,
|
||||||
BaseUserManager,
|
BaseUserManager,
|
||||||
PermissionsMixin,
|
PermissionsMixin,
|
||||||
)
|
)
|
||||||
from django.db import models
|
from django.db import models
|
||||||
|
from django_celery_beat.models import PeriodicTask
|
||||||
|
|
||||||
|
|
||||||
class AccountManager(BaseUserManager):
|
class AccountManager(BaseUserManager):
|
||||||
|
@ -51,3 +53,9 @@ class Account(AbstractBaseUser, PermissionsMixin):
|
||||||
|
|
||||||
USERNAME_FIELD = "name"
|
USERNAME_FIELD = "name"
|
||||||
REQUIRED_FIELDS = ["password"]
|
REQUIRED_FIELDS = ["password"]
|
||||||
|
|
||||||
|
|
||||||
|
class CustomPeriodicTask(PeriodicTask):
|
||||||
|
"""add custom metadata to to task"""
|
||||||
|
|
||||||
|
task_config = models.JSONField(default=dict)
|
||||||
|
|
|
@ -7,18 +7,14 @@ Functionality:
|
||||||
import json
|
import json
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
from home.src.download.subscriptions import (
|
from home.src.download.subscriptions import ChannelSubscription
|
||||||
ChannelSubscription,
|
|
||||||
PlaylistSubscription,
|
|
||||||
)
|
|
||||||
from home.src.download.thumbnails import ThumbManager
|
from home.src.download.thumbnails import ThumbManager
|
||||||
from home.src.download.yt_dlp_base import YtWrap
|
from home.src.download.yt_dlp_base import YtWrap
|
||||||
from home.src.es.connect import ElasticWrap, IndexPaginate
|
from home.src.es.connect import ElasticWrap, IndexPaginate
|
||||||
from home.src.index.playlist import YoutubePlaylist
|
from home.src.index.playlist import YoutubePlaylist
|
||||||
from home.src.index.video_constants import VideoTypeEnum
|
from home.src.index.video_constants import VideoTypeEnum
|
||||||
from home.src.ta.config import AppConfig
|
from home.src.ta.config import AppConfig
|
||||||
from home.src.ta.helper import DurationConverter, is_shorts
|
from home.src.ta.helper import get_duration_str, is_shorts
|
||||||
from home.src.ta.ta_redis import RedisQueue
|
|
||||||
|
|
||||||
|
|
||||||
class PendingIndex:
|
class PendingIndex:
|
||||||
|
@ -112,20 +108,20 @@ class PendingInteract:
|
||||||
_, _ = ElasticWrap(path).post(data=data)
|
_, _ = ElasticWrap(path).post(data=data)
|
||||||
|
|
||||||
def update_status(self):
|
def update_status(self):
|
||||||
"""update status field of pending item"""
|
"""update status of pending item"""
|
||||||
data = {"doc": {"status": self.status}}
|
if self.status == "priority":
|
||||||
path = f"ta_download/_update/{self.youtube_id}"
|
data = {
|
||||||
_, _ = ElasticWrap(path).post(data=data)
|
"doc": {
|
||||||
|
"status": "pending",
|
||||||
|
"auto_start": True,
|
||||||
|
"message": None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
data = {"doc": {"status": self.status}}
|
||||||
|
|
||||||
def prioritize(self):
|
path = f"ta_download/_update/{self.youtube_id}/?refresh=true"
|
||||||
"""prioritize pending item in redis queue"""
|
_, _ = ElasticWrap(path).post(data=data)
|
||||||
pending_video, _ = self.get_item()
|
|
||||||
vid_type = pending_video.get("vid_type", VideoTypeEnum.VIDEOS.value)
|
|
||||||
to_add = {
|
|
||||||
"youtube_id": pending_video["youtube_id"],
|
|
||||||
"vid_type": vid_type,
|
|
||||||
}
|
|
||||||
RedisQueue(queue_name="dl_queue").add_priority(to_add)
|
|
||||||
|
|
||||||
def get_item(self):
|
def get_item(self):
|
||||||
"""return pending item dict"""
|
"""return pending item dict"""
|
||||||
|
@ -197,7 +193,6 @@ class PendingList(PendingIndex):
|
||||||
self._parse_channel(entry["url"], vid_type)
|
self._parse_channel(entry["url"], vid_type)
|
||||||
elif entry["type"] == "playlist":
|
elif entry["type"] == "playlist":
|
||||||
self._parse_playlist(entry["url"])
|
self._parse_playlist(entry["url"])
|
||||||
PlaylistSubscription().process_url_str([entry], subscribed=False)
|
|
||||||
else:
|
else:
|
||||||
raise ValueError(f"invalid url_type: {entry}")
|
raise ValueError(f"invalid url_type: {entry}")
|
||||||
|
|
||||||
|
@ -228,19 +223,28 @@ class PendingList(PendingIndex):
|
||||||
def _parse_playlist(self, url):
|
def _parse_playlist(self, url):
|
||||||
"""add all videos of playlist to list"""
|
"""add all videos of playlist to list"""
|
||||||
playlist = YoutubePlaylist(url)
|
playlist = YoutubePlaylist(url)
|
||||||
playlist.build_json()
|
is_active = playlist.update_playlist()
|
||||||
video_results = playlist.json_data.get("playlist_entries")
|
if not is_active:
|
||||||
youtube_ids = [i["youtube_id"] for i in video_results]
|
message = f"{playlist.youtube_id}: failed to extract metadata"
|
||||||
for video_id in youtube_ids:
|
print(message)
|
||||||
|
raise ValueError(message)
|
||||||
|
|
||||||
|
entries = playlist.json_data["playlist_entries"]
|
||||||
|
to_add = [i["youtube_id"] for i in entries if not i["downloaded"]]
|
||||||
|
if not to_add:
|
||||||
|
return
|
||||||
|
|
||||||
|
for video_id in to_add:
|
||||||
# match vid_type later
|
# match vid_type later
|
||||||
self._add_video(video_id, VideoTypeEnum.UNKNOWN)
|
self._add_video(video_id, VideoTypeEnum.UNKNOWN)
|
||||||
|
|
||||||
def add_to_pending(self, status="pending"):
|
def add_to_pending(self, status="pending", auto_start=False):
|
||||||
"""add missing videos to pending list"""
|
"""add missing videos to pending list"""
|
||||||
self.get_channels()
|
self.get_channels()
|
||||||
bulk_list = []
|
bulk_list = []
|
||||||
|
|
||||||
total = len(self.missing_videos)
|
total = len(self.missing_videos)
|
||||||
|
videos_added = []
|
||||||
for idx, (youtube_id, vid_type) in enumerate(self.missing_videos):
|
for idx, (youtube_id, vid_type) in enumerate(self.missing_videos):
|
||||||
if self.task and self.task.is_stopped():
|
if self.task and self.task.is_stopped():
|
||||||
break
|
break
|
||||||
|
@ -251,13 +255,20 @@ class PendingList(PendingIndex):
|
||||||
if not video_details:
|
if not video_details:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
video_details["status"] = status
|
video_details.update(
|
||||||
|
{
|
||||||
|
"status": status,
|
||||||
|
"auto_start": auto_start,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
action = {"create": {"_id": youtube_id, "_index": "ta_download"}}
|
action = {"create": {"_id": youtube_id, "_index": "ta_download"}}
|
||||||
bulk_list.append(json.dumps(action))
|
bulk_list.append(json.dumps(action))
|
||||||
bulk_list.append(json.dumps(video_details))
|
bulk_list.append(json.dumps(video_details))
|
||||||
|
|
||||||
url = video_details["vid_thumb_url"]
|
url = video_details["vid_thumb_url"]
|
||||||
ThumbManager(youtube_id).download_video_thumb(url)
|
ThumbManager(youtube_id).download_video_thumb(url)
|
||||||
|
videos_added.append(youtube_id)
|
||||||
|
|
||||||
if len(bulk_list) >= 20:
|
if len(bulk_list) >= 20:
|
||||||
self._ingest_bulk(bulk_list)
|
self._ingest_bulk(bulk_list)
|
||||||
|
@ -265,6 +276,8 @@ class PendingList(PendingIndex):
|
||||||
|
|
||||||
self._ingest_bulk(bulk_list)
|
self._ingest_bulk(bulk_list)
|
||||||
|
|
||||||
|
return videos_added
|
||||||
|
|
||||||
def _ingest_bulk(self, bulk_list):
|
def _ingest_bulk(self, bulk_list):
|
||||||
"""add items to queue in bulk"""
|
"""add items to queue in bulk"""
|
||||||
if not bulk_list:
|
if not bulk_list:
|
||||||
|
@ -273,7 +286,7 @@ class PendingList(PendingIndex):
|
||||||
# add last newline
|
# add last newline
|
||||||
bulk_list.append("\n")
|
bulk_list.append("\n")
|
||||||
query_str = "\n".join(bulk_list)
|
query_str = "\n".join(bulk_list)
|
||||||
_, _ = ElasticWrap("_bulk").post(query_str, ndjson=True)
|
_, _ = ElasticWrap("_bulk?refresh=true").post(query_str, ndjson=True)
|
||||||
|
|
||||||
def _notify_add(self, idx, total):
|
def _notify_add(self, idx, total):
|
||||||
"""send notification for adding videos to download queue"""
|
"""send notification for adding videos to download queue"""
|
||||||
|
@ -329,9 +342,6 @@ class PendingList(PendingIndex):
|
||||||
def _parse_youtube_details(self, vid, vid_type=VideoTypeEnum.VIDEOS):
|
def _parse_youtube_details(self, vid, vid_type=VideoTypeEnum.VIDEOS):
|
||||||
"""parse response"""
|
"""parse response"""
|
||||||
vid_id = vid.get("id")
|
vid_id = vid.get("id")
|
||||||
duration_str = DurationConverter.get_str(vid["duration"])
|
|
||||||
if duration_str == "NA":
|
|
||||||
print(f"skip extracting duration for: {vid_id}")
|
|
||||||
published = datetime.strptime(vid["upload_date"], "%Y%m%d").strftime(
|
published = datetime.strptime(vid["upload_date"], "%Y%m%d").strftime(
|
||||||
"%Y-%m-%d"
|
"%Y-%m-%d"
|
||||||
)
|
)
|
||||||
|
@ -343,7 +353,7 @@ class PendingList(PendingIndex):
|
||||||
"vid_thumb_url": vid["thumbnail"],
|
"vid_thumb_url": vid["thumbnail"],
|
||||||
"title": vid["title"],
|
"title": vid["title"],
|
||||||
"channel_id": vid["channel_id"],
|
"channel_id": vid["channel_id"],
|
||||||
"duration": duration_str,
|
"duration": get_duration_str(vid["duration"]),
|
||||||
"published": published,
|
"published": published,
|
||||||
"timestamp": int(datetime.now().timestamp()),
|
"timestamp": int(datetime.now().timestamp()),
|
||||||
# Pulling enum value out so it is serializable
|
# Pulling enum value out so it is serializable
|
||||||
|
|
|
@ -4,15 +4,15 @@ Functionality:
|
||||||
- handle playlist subscriptions
|
- handle playlist subscriptions
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from home.src.download import queue # partial import
|
|
||||||
from home.src.download.thumbnails import ThumbManager
|
from home.src.download.thumbnails import ThumbManager
|
||||||
from home.src.download.yt_dlp_base import YtWrap
|
from home.src.download.yt_dlp_base import YtWrap
|
||||||
from home.src.es.connect import IndexPaginate
|
from home.src.es.connect import IndexPaginate
|
||||||
from home.src.index.channel import YoutubeChannel
|
from home.src.index.channel import YoutubeChannel
|
||||||
from home.src.index.playlist import YoutubePlaylist
|
from home.src.index.playlist import YoutubePlaylist
|
||||||
|
from home.src.index.video import YoutubeVideo
|
||||||
from home.src.index.video_constants import VideoTypeEnum
|
from home.src.index.video_constants import VideoTypeEnum
|
||||||
from home.src.ta.config import AppConfig
|
from home.src.ta.config import AppConfig
|
||||||
from home.src.ta.ta_redis import RedisArchivist
|
from home.src.ta.helper import is_missing
|
||||||
from home.src.ta.urlparser import Parser
|
from home.src.ta.urlparser import Parser
|
||||||
|
|
||||||
|
|
||||||
|
@ -106,10 +106,6 @@ class ChannelSubscription:
|
||||||
if not all_channels:
|
if not all_channels:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
pending = queue.PendingList()
|
|
||||||
pending.get_download()
|
|
||||||
pending.get_indexed()
|
|
||||||
|
|
||||||
missing_videos = []
|
missing_videos = []
|
||||||
|
|
||||||
total = len(all_channels)
|
total = len(all_channels)
|
||||||
|
@ -119,22 +115,22 @@ class ChannelSubscription:
|
||||||
last_videos = self.get_last_youtube_videos(channel_id)
|
last_videos = self.get_last_youtube_videos(channel_id)
|
||||||
|
|
||||||
if last_videos:
|
if last_videos:
|
||||||
|
ids_to_add = is_missing([i[0] for i in last_videos])
|
||||||
for video_id, _, vid_type in last_videos:
|
for video_id, _, vid_type in last_videos:
|
||||||
if video_id not in pending.to_skip:
|
if video_id in ids_to_add:
|
||||||
missing_videos.append((video_id, vid_type))
|
missing_videos.append((video_id, vid_type))
|
||||||
|
|
||||||
if not self.task:
|
if not self.task:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if self.task:
|
if self.task.is_stopped():
|
||||||
if self.task.is_stopped():
|
self.task.send_progress(["Received Stop signal."])
|
||||||
self.task.send_progress(["Received Stop signal."])
|
break
|
||||||
break
|
|
||||||
|
|
||||||
self.task.send_progress(
|
self.task.send_progress(
|
||||||
message_lines=[f"Scanning Channel {idx + 1}/{total}"],
|
message_lines=[f"Scanning Channel {idx + 1}/{total}"],
|
||||||
progress=(idx + 1) / total,
|
progress=(idx + 1) / total,
|
||||||
)
|
)
|
||||||
|
|
||||||
return missing_videos
|
return missing_videos
|
||||||
|
|
||||||
|
@ -175,13 +171,6 @@ class PlaylistSubscription:
|
||||||
|
|
||||||
def process_url_str(self, new_playlists, subscribed=True):
|
def process_url_str(self, new_playlists, subscribed=True):
|
||||||
"""process playlist subscribe form url_str"""
|
"""process playlist subscribe form url_str"""
|
||||||
data = {
|
|
||||||
"query": {"match_all": {}},
|
|
||||||
"sort": [{"published": {"order": "desc"}}],
|
|
||||||
}
|
|
||||||
all_indexed = IndexPaginate("ta_video", data).get_results()
|
|
||||||
all_youtube_ids = [i["youtube_id"] for i in all_indexed]
|
|
||||||
|
|
||||||
for idx, playlist in enumerate(new_playlists):
|
for idx, playlist in enumerate(new_playlists):
|
||||||
playlist_id = playlist["url"]
|
playlist_id = playlist["url"]
|
||||||
if not playlist["type"] == "playlist":
|
if not playlist["type"] == "playlist":
|
||||||
|
@ -189,8 +178,12 @@ class PlaylistSubscription:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
playlist_h = YoutubePlaylist(playlist_id)
|
playlist_h = YoutubePlaylist(playlist_id)
|
||||||
playlist_h.all_youtube_ids = all_youtube_ids
|
|
||||||
playlist_h.build_json()
|
playlist_h.build_json()
|
||||||
|
if not playlist_h.json_data:
|
||||||
|
message = f"{playlist_h.youtube_id}: failed to extract data"
|
||||||
|
print(message)
|
||||||
|
raise ValueError(message)
|
||||||
|
|
||||||
playlist_h.json_data["playlist_subscribed"] = subscribed
|
playlist_h.json_data["playlist_subscribed"] = subscribed
|
||||||
playlist_h.upload_to_es()
|
playlist_h.upload_to_es()
|
||||||
playlist_h.add_vids_to_playlist()
|
playlist_h.add_vids_to_playlist()
|
||||||
|
@ -200,16 +193,13 @@ class PlaylistSubscription:
|
||||||
thumb = ThumbManager(playlist_id, item_type="playlist")
|
thumb = ThumbManager(playlist_id, item_type="playlist")
|
||||||
thumb.download_playlist_thumb(url)
|
thumb.download_playlist_thumb(url)
|
||||||
|
|
||||||
# notify
|
if self.task:
|
||||||
message = {
|
self.task.send_progress(
|
||||||
"status": "message:subplaylist",
|
message_lines=[
|
||||||
"level": "info",
|
f"Processing {idx + 1} of {len(new_playlists)}"
|
||||||
"title": "Subscribing to Playlists",
|
],
|
||||||
"message": f"Processing {idx + 1} of {len(new_playlists)}",
|
progress=(idx + 1) / len(new_playlists),
|
||||||
}
|
)
|
||||||
RedisArchivist().set_message(
|
|
||||||
"message:subplaylist", message=message, expire=True
|
|
||||||
)
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def channel_validate(channel_id):
|
def channel_validate(channel_id):
|
||||||
|
@ -225,27 +215,15 @@ class PlaylistSubscription:
|
||||||
playlist.json_data["playlist_subscribed"] = subscribe_status
|
playlist.json_data["playlist_subscribed"] = subscribe_status
|
||||||
playlist.upload_to_es()
|
playlist.upload_to_es()
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def get_to_ignore():
|
|
||||||
"""get all youtube_ids already downloaded or ignored"""
|
|
||||||
pending = queue.PendingList()
|
|
||||||
pending.get_download()
|
|
||||||
pending.get_indexed()
|
|
||||||
|
|
||||||
return pending.to_skip
|
|
||||||
|
|
||||||
def find_missing(self):
|
def find_missing(self):
|
||||||
"""find videos in subscribed playlists not downloaded yet"""
|
"""find videos in subscribed playlists not downloaded yet"""
|
||||||
all_playlists = [i["playlist_id"] for i in self.get_playlists()]
|
all_playlists = [i["playlist_id"] for i in self.get_playlists()]
|
||||||
if not all_playlists:
|
if not all_playlists:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
to_ignore = self.get_to_ignore()
|
|
||||||
|
|
||||||
missing_videos = []
|
missing_videos = []
|
||||||
total = len(all_playlists)
|
total = len(all_playlists)
|
||||||
for idx, playlist_id in enumerate(all_playlists):
|
for idx, playlist_id in enumerate(all_playlists):
|
||||||
size_limit = self.config["subscriptions"]["channel_size"]
|
|
||||||
playlist = YoutubePlaylist(playlist_id)
|
playlist = YoutubePlaylist(playlist_id)
|
||||||
is_active = playlist.update_playlist()
|
is_active = playlist.update_playlist()
|
||||||
if not is_active:
|
if not is_active:
|
||||||
|
@ -253,27 +231,29 @@ class PlaylistSubscription:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
playlist_entries = playlist.json_data["playlist_entries"]
|
playlist_entries = playlist.json_data["playlist_entries"]
|
||||||
|
size_limit = self.config["subscriptions"]["channel_size"]
|
||||||
if size_limit:
|
if size_limit:
|
||||||
del playlist_entries[size_limit:]
|
del playlist_entries[size_limit:]
|
||||||
|
|
||||||
all_missing = [i for i in playlist_entries if not i["downloaded"]]
|
to_check = [
|
||||||
|
i["youtube_id"]
|
||||||
for video in all_missing:
|
for i in playlist_entries
|
||||||
youtube_id = video["youtube_id"]
|
if i["downloaded"] is False
|
||||||
if youtube_id not in to_ignore:
|
]
|
||||||
missing_videos.append(youtube_id)
|
needs_downloading = is_missing(to_check)
|
||||||
|
missing_videos.extend(needs_downloading)
|
||||||
|
|
||||||
if not self.task:
|
if not self.task:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if self.task:
|
if self.task.is_stopped():
|
||||||
self.task.send_progress(
|
self.task.send_progress(["Received Stop signal."])
|
||||||
message_lines=[f"Scanning Playlists {idx + 1}/{total}"],
|
break
|
||||||
progress=(idx + 1) / total,
|
|
||||||
)
|
self.task.send_progress(
|
||||||
if self.task.is_stopped():
|
message_lines=[f"Scanning Playlists {idx + 1}/{total}"],
|
||||||
self.task.send_progress(["Received Stop signal."])
|
progress=(idx + 1) / total,
|
||||||
break
|
)
|
||||||
|
|
||||||
return missing_videos
|
return missing_videos
|
||||||
|
|
||||||
|
@ -284,6 +264,7 @@ class SubscriptionScanner:
|
||||||
def __init__(self, task=False):
|
def __init__(self, task=False):
|
||||||
self.task = task
|
self.task = task
|
||||||
self.missing_videos = False
|
self.missing_videos = False
|
||||||
|
self.auto_start = AppConfig().config["subscriptions"].get("auto_start")
|
||||||
|
|
||||||
def scan(self):
|
def scan(self):
|
||||||
"""scan channels and playlists"""
|
"""scan channels and playlists"""
|
||||||
|
@ -334,7 +315,7 @@ class SubscriptionHandler:
|
||||||
self.task = task
|
self.task = task
|
||||||
self.to_subscribe = False
|
self.to_subscribe = False
|
||||||
|
|
||||||
def subscribe(self):
|
def subscribe(self, expected_type=False):
|
||||||
"""subscribe to url_str items"""
|
"""subscribe to url_str items"""
|
||||||
if self.task:
|
if self.task:
|
||||||
self.task.send_progress(["Processing form content."])
|
self.task.send_progress(["Processing form content."])
|
||||||
|
@ -345,23 +326,35 @@ class SubscriptionHandler:
|
||||||
if self.task:
|
if self.task:
|
||||||
self._notify(idx, item, total)
|
self._notify(idx, item, total)
|
||||||
|
|
||||||
self.subscribe_type(item)
|
self.subscribe_type(item, expected_type=expected_type)
|
||||||
|
|
||||||
def subscribe_type(self, item):
|
def subscribe_type(self, item, expected_type):
|
||||||
"""process single item"""
|
"""process single item"""
|
||||||
if item["type"] == "playlist":
|
if item["type"] == "playlist":
|
||||||
|
if expected_type and expected_type != "playlist":
|
||||||
|
raise TypeError(
|
||||||
|
f"expected {expected_type} url but got {item.get('type')}"
|
||||||
|
)
|
||||||
|
|
||||||
PlaylistSubscription().process_url_str([item])
|
PlaylistSubscription().process_url_str([item])
|
||||||
return
|
return
|
||||||
|
|
||||||
if item["type"] == "video":
|
if item["type"] == "video":
|
||||||
# extract channel id from video
|
# extract channel id from video
|
||||||
vid = queue.PendingList().get_youtube_details(item["url"])
|
video = YoutubeVideo(item["url"])
|
||||||
channel_id = vid["channel_id"]
|
video.get_from_youtube()
|
||||||
|
video.process_youtube_meta()
|
||||||
|
channel_id = video.channel_id
|
||||||
elif item["type"] == "channel":
|
elif item["type"] == "channel":
|
||||||
channel_id = item["url"]
|
channel_id = item["url"]
|
||||||
else:
|
else:
|
||||||
raise ValueError("failed to subscribe to: " + item["url"])
|
raise ValueError("failed to subscribe to: " + item["url"])
|
||||||
|
|
||||||
|
if expected_type and expected_type != "channel":
|
||||||
|
raise TypeError(
|
||||||
|
f"expected {expected_type} url but got {item.get('type')}"
|
||||||
|
)
|
||||||
|
|
||||||
self._subscribe(channel_id)
|
self._subscribe(channel_id)
|
||||||
|
|
||||||
def _subscribe(self, channel_id):
|
def _subscribe(self, channel_id):
|
||||||
|
|
|
@ -11,7 +11,8 @@ from time import sleep
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
from home.src.es.connect import ElasticWrap, IndexPaginate
|
from home.src.es.connect import ElasticWrap, IndexPaginate
|
||||||
from home.src.ta.config import AppConfig
|
from home.src.ta.helper import is_missing
|
||||||
|
from home.src.ta.settings import EnvironmentSettings
|
||||||
from mutagen.mp4 import MP4, MP4Cover
|
from mutagen.mp4 import MP4, MP4Cover
|
||||||
from PIL import Image, ImageFile, ImageFilter, UnidentifiedImageError
|
from PIL import Image, ImageFile, ImageFilter, UnidentifiedImageError
|
||||||
|
|
||||||
|
@ -21,8 +22,7 @@ ImageFile.LOAD_TRUNCATED_IMAGES = True
|
||||||
class ThumbManagerBase:
|
class ThumbManagerBase:
|
||||||
"""base class for thumbnail management"""
|
"""base class for thumbnail management"""
|
||||||
|
|
||||||
CONFIG = AppConfig().config
|
CACHE_DIR = EnvironmentSettings.CACHE_DIR
|
||||||
CACHE_DIR = CONFIG["application"]["cache_dir"]
|
|
||||||
VIDEO_DIR = os.path.join(CACHE_DIR, "videos")
|
VIDEO_DIR = os.path.join(CACHE_DIR, "videos")
|
||||||
CHANNEL_DIR = os.path.join(CACHE_DIR, "channels")
|
CHANNEL_DIR = os.path.join(CACHE_DIR, "channels")
|
||||||
PLAYLIST_DIR = os.path.join(CACHE_DIR, "playlists")
|
PLAYLIST_DIR = os.path.join(CACHE_DIR, "playlists")
|
||||||
|
@ -54,11 +54,14 @@ class ThumbManagerBase:
|
||||||
if response.status_code == 404:
|
if response.status_code == 404:
|
||||||
return self.get_fallback()
|
return self.get_fallback()
|
||||||
|
|
||||||
except requests.exceptions.RequestException:
|
except (
|
||||||
|
requests.exceptions.RequestException,
|
||||||
|
requests.exceptions.ReadTimeout,
|
||||||
|
):
|
||||||
print(f"{self.item_id}: retry thumbnail download {url}")
|
print(f"{self.item_id}: retry thumbnail download {url}")
|
||||||
sleep((i + 1) ** i)
|
sleep((i + 1) ** i)
|
||||||
|
|
||||||
return False
|
return self.get_fallback()
|
||||||
|
|
||||||
def get_fallback(self):
|
def get_fallback(self):
|
||||||
"""get fallback thumbnail if not available"""
|
"""get fallback thumbnail if not available"""
|
||||||
|
@ -67,13 +70,13 @@ class ThumbManagerBase:
|
||||||
img_raw = Image.open(self.fallback)
|
img_raw = Image.open(self.fallback)
|
||||||
return img_raw
|
return img_raw
|
||||||
|
|
||||||
app_root = self.CONFIG["application"]["app_root"]
|
app_root = EnvironmentSettings.APP_DIR
|
||||||
default_map = {
|
default_map = {
|
||||||
"video": os.path.join(
|
"video": os.path.join(
|
||||||
app_root, "static/img/default-video-thumb.jpg"
|
app_root, "static/img/default-video-thumb.jpg"
|
||||||
),
|
),
|
||||||
"playlist": os.path.join(
|
"playlist": os.path.join(
|
||||||
app_root, "static/img/default-video-thumb.jpg"
|
app_root, "static/img/default-playlist-thumb.jpg"
|
||||||
),
|
),
|
||||||
"icon": os.path.join(
|
"icon": os.path.join(
|
||||||
app_root, "static/img/default-channel-icon.jpg"
|
app_root, "static/img/default-channel-icon.jpg"
|
||||||
|
@ -200,7 +203,18 @@ class ThumbManager(ThumbManagerBase):
|
||||||
if skip_existing and os.path.exists(thumb_path):
|
if skip_existing and os.path.exists(thumb_path):
|
||||||
return
|
return
|
||||||
|
|
||||||
img_raw = self.download_raw(url)
|
img_raw = (
|
||||||
|
self.download_raw(url)
|
||||||
|
if not isinstance(url, str) or url.startswith("http")
|
||||||
|
else Image.open(os.path.join(self.CACHE_DIR, url))
|
||||||
|
)
|
||||||
|
width, height = img_raw.size
|
||||||
|
|
||||||
|
if not width / height == 16 / 9:
|
||||||
|
new_height = width / 16 * 9
|
||||||
|
offset = (height - new_height) / 2
|
||||||
|
img_raw = img_raw.crop((0, offset, width, height - offset))
|
||||||
|
img_raw = img_raw.resize((336, 189))
|
||||||
img_raw.convert("RGB").save(thumb_path)
|
img_raw.convert("RGB").save(thumb_path)
|
||||||
|
|
||||||
def delete_video_thumb(self):
|
def delete_video_thumb(self):
|
||||||
|
@ -243,9 +257,10 @@ class ThumbManager(ThumbManagerBase):
|
||||||
class ValidatorCallback:
|
class ValidatorCallback:
|
||||||
"""handle callback validate thumbnails page by page"""
|
"""handle callback validate thumbnails page by page"""
|
||||||
|
|
||||||
def __init__(self, source, index_name):
|
def __init__(self, source, index_name, counter=0):
|
||||||
self.source = source
|
self.source = source
|
||||||
self.index_name = index_name
|
self.index_name = index_name
|
||||||
|
self.counter = counter
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
"""run the task for page"""
|
"""run the task for page"""
|
||||||
|
@ -270,7 +285,7 @@ class ValidatorCallback:
|
||||||
urls = (
|
urls = (
|
||||||
channel["_source"]["channel_thumb_url"],
|
channel["_source"]["channel_thumb_url"],
|
||||||
channel["_source"]["channel_banner_url"],
|
channel["_source"]["channel_banner_url"],
|
||||||
channel["_source"]["channel_tvart_url"],
|
channel["_source"].get("channel_tvart_url", False),
|
||||||
)
|
)
|
||||||
handler = ThumbManager(channel["_source"]["channel_id"])
|
handler = ThumbManager(channel["_source"]["channel_id"])
|
||||||
handler.download_channel_art(urls, skip_existing=True)
|
handler.download_channel_art(urls, skip_existing=True)
|
||||||
|
@ -312,7 +327,7 @@ class ThumbValidator:
|
||||||
},
|
},
|
||||||
]
|
]
|
||||||
|
|
||||||
def __init__(self, task):
|
def __init__(self, task=False):
|
||||||
self.task = task
|
self.task = task
|
||||||
|
|
||||||
def validate(self):
|
def validate(self):
|
||||||
|
@ -332,6 +347,89 @@ class ThumbValidator:
|
||||||
)
|
)
|
||||||
_ = paginate.get_results()
|
_ = paginate.get_results()
|
||||||
|
|
||||||
|
def clean_up(self):
|
||||||
|
"""clean up all thumbs"""
|
||||||
|
self._clean_up_vids()
|
||||||
|
self._clean_up_channels()
|
||||||
|
self._clean_up_playlists()
|
||||||
|
|
||||||
|
def _clean_up_vids(self):
|
||||||
|
"""clean unneeded vid thumbs"""
|
||||||
|
video_dir = os.path.join(EnvironmentSettings.CACHE_DIR, "videos")
|
||||||
|
video_folders = os.listdir(video_dir)
|
||||||
|
for video_folder in video_folders:
|
||||||
|
folder_path = os.path.join(video_dir, video_folder)
|
||||||
|
thumbs_is = {i.split(".")[0] for i in os.listdir(folder_path)}
|
||||||
|
thumbs_should = self._get_vid_thumbs_should(video_folder)
|
||||||
|
to_delete = thumbs_is - thumbs_should
|
||||||
|
for thumb in to_delete:
|
||||||
|
delete_path = os.path.join(folder_path, f"{thumb}.jpg")
|
||||||
|
os.remove(delete_path)
|
||||||
|
|
||||||
|
if to_delete:
|
||||||
|
message = (
|
||||||
|
f"[thumbs][video][{video_folder}] "
|
||||||
|
+ f"delete {len(to_delete)} unused thumbnails"
|
||||||
|
)
|
||||||
|
print(message)
|
||||||
|
if self.task:
|
||||||
|
self.task.send_progress([message])
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _get_vid_thumbs_should(video_folder: str) -> set[str]:
|
||||||
|
"""get indexed"""
|
||||||
|
should_list = [
|
||||||
|
{"prefix": {"youtube_id": {"value": video_folder.lower()}}},
|
||||||
|
{"prefix": {"youtube_id": {"value": video_folder.upper()}}},
|
||||||
|
]
|
||||||
|
data = {
|
||||||
|
"query": {"bool": {"should": should_list}},
|
||||||
|
"_source": ["youtube_id"],
|
||||||
|
}
|
||||||
|
result = IndexPaginate("ta_video,ta_download", data).get_results()
|
||||||
|
thumbs_should = {i["youtube_id"] for i in result}
|
||||||
|
|
||||||
|
return thumbs_should
|
||||||
|
|
||||||
|
def _clean_up_channels(self):
|
||||||
|
"""clean unneeded channel thumbs"""
|
||||||
|
channel_dir = os.path.join(EnvironmentSettings.CACHE_DIR, "channels")
|
||||||
|
channel_art = os.listdir(channel_dir)
|
||||||
|
thumbs_is = {"_".join(i.split("_")[:-1]) for i in channel_art}
|
||||||
|
to_delete = is_missing(list(thumbs_is), "ta_channel", "channel_id")
|
||||||
|
for channel_thumb in channel_art:
|
||||||
|
if channel_thumb[:24] in to_delete:
|
||||||
|
delete_path = os.path.join(channel_dir, channel_thumb)
|
||||||
|
os.remove(delete_path)
|
||||||
|
|
||||||
|
if to_delete:
|
||||||
|
message = (
|
||||||
|
"[thumbs][channel] "
|
||||||
|
+ f"delete {len(to_delete)} unused channel art"
|
||||||
|
)
|
||||||
|
print(message)
|
||||||
|
if self.task:
|
||||||
|
self.task.send_progress([message])
|
||||||
|
|
||||||
|
def _clean_up_playlists(self):
|
||||||
|
"""clean up unneeded playlist thumbs"""
|
||||||
|
playlist_dir = os.path.join(EnvironmentSettings.CACHE_DIR, "playlists")
|
||||||
|
playlist_art = os.listdir(playlist_dir)
|
||||||
|
thumbs_is = {i.split(".")[0] for i in playlist_art}
|
||||||
|
to_delete = is_missing(list(thumbs_is), "ta_playlist", "playlist_id")
|
||||||
|
for playlist_id in to_delete:
|
||||||
|
delete_path = os.path.join(playlist_dir, f"{playlist_id}.jpg")
|
||||||
|
os.remove(delete_path)
|
||||||
|
|
||||||
|
if to_delete:
|
||||||
|
message = (
|
||||||
|
"[thumbs][playlist] "
|
||||||
|
+ f"delete {len(to_delete)} unused playlist art"
|
||||||
|
)
|
||||||
|
print(message)
|
||||||
|
if self.task:
|
||||||
|
self.task.send_progress([message])
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _get_total(index_name):
|
def _get_total(index_name):
|
||||||
"""get total documents in index"""
|
"""get total documents in index"""
|
||||||
|
@ -376,14 +474,14 @@ class ThumbFilesystem:
|
||||||
class EmbedCallback:
|
class EmbedCallback:
|
||||||
"""callback class to embed thumbnails"""
|
"""callback class to embed thumbnails"""
|
||||||
|
|
||||||
CONFIG = AppConfig().config
|
CACHE_DIR = EnvironmentSettings.CACHE_DIR
|
||||||
CACHE_DIR = CONFIG["application"]["cache_dir"]
|
MEDIA_DIR = EnvironmentSettings.MEDIA_DIR
|
||||||
MEDIA_DIR = CONFIG["application"]["videos"]
|
|
||||||
FORMAT = MP4Cover.FORMAT_JPEG
|
FORMAT = MP4Cover.FORMAT_JPEG
|
||||||
|
|
||||||
def __init__(self, source, index_name):
|
def __init__(self, source, index_name, counter=0):
|
||||||
self.source = source
|
self.source = source
|
||||||
self.index_name = index_name
|
self.index_name = index_name
|
||||||
|
self.counter = counter
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
"""run embed"""
|
"""run embed"""
|
||||||
|
|
|
@ -10,6 +10,7 @@ from http import cookiejar
|
||||||
from io import StringIO
|
from io import StringIO
|
||||||
|
|
||||||
import yt_dlp
|
import yt_dlp
|
||||||
|
from home.src.ta.settings import EnvironmentSettings
|
||||||
from home.src.ta.ta_redis import RedisArchivist
|
from home.src.ta.ta_redis import RedisArchivist
|
||||||
|
|
||||||
|
|
||||||
|
@ -48,21 +49,33 @@ class YtWrap:
|
||||||
with yt_dlp.YoutubeDL(self.obs) as ydl:
|
with yt_dlp.YoutubeDL(self.obs) as ydl:
|
||||||
try:
|
try:
|
||||||
ydl.download([url])
|
ydl.download([url])
|
||||||
except yt_dlp.utils.DownloadError:
|
except yt_dlp.utils.DownloadError as err:
|
||||||
print(f"{url}: failed to download.")
|
print(f"{url}: failed to download with message {err}")
|
||||||
return False
|
if "Temporary failure in name resolution" in str(err):
|
||||||
|
raise ConnectionError("lost the internet, abort!") from err
|
||||||
|
|
||||||
return True
|
return False, str(err)
|
||||||
|
|
||||||
|
return True, True
|
||||||
|
|
||||||
def extract(self, url):
|
def extract(self, url):
|
||||||
"""make extract request"""
|
"""make extract request"""
|
||||||
try:
|
try:
|
||||||
response = yt_dlp.YoutubeDL(self.obs).extract_info(url)
|
response = yt_dlp.YoutubeDL(self.obs).extract_info(url)
|
||||||
except cookiejar.LoadError:
|
except cookiejar.LoadError as err:
|
||||||
print("cookie file is invalid")
|
print(f"cookie file is invalid: {err}")
|
||||||
return False
|
return False
|
||||||
except (yt_dlp.utils.ExtractorError, yt_dlp.utils.DownloadError):
|
except yt_dlp.utils.ExtractorError as err:
|
||||||
print(f"{url}: failed to get info from youtube")
|
print(f"{url}: failed to extract with message: {err}, continue...")
|
||||||
|
return False
|
||||||
|
except yt_dlp.utils.DownloadError as err:
|
||||||
|
if "This channel does not have a" in str(err):
|
||||||
|
return False
|
||||||
|
|
||||||
|
print(f"{url}: failed to get info from youtube with message {err}")
|
||||||
|
if "Temporary failure in name resolution" in str(err):
|
||||||
|
raise ConnectionError("lost the internet, abort!") from err
|
||||||
|
|
||||||
return False
|
return False
|
||||||
|
|
||||||
return response
|
return response
|
||||||
|
@ -74,6 +87,7 @@ class CookieHandler:
|
||||||
def __init__(self, config):
|
def __init__(self, config):
|
||||||
self.cookie_io = False
|
self.cookie_io = False
|
||||||
self.config = config
|
self.config = config
|
||||||
|
self.cache_dir = EnvironmentSettings.CACHE_DIR
|
||||||
|
|
||||||
def get(self):
|
def get(self):
|
||||||
"""get cookie io stream"""
|
"""get cookie io stream"""
|
||||||
|
@ -83,8 +97,9 @@ class CookieHandler:
|
||||||
|
|
||||||
def import_cookie(self):
|
def import_cookie(self):
|
||||||
"""import cookie from file"""
|
"""import cookie from file"""
|
||||||
cache_path = self.config["application"]["cache_dir"]
|
import_path = os.path.join(
|
||||||
import_path = os.path.join(cache_path, "import", "cookies.google.txt")
|
self.cache_dir, "import", "cookies.google.txt"
|
||||||
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
with open(import_path, encoding="utf-8") as cookie_file:
|
with open(import_path, encoding="utf-8") as cookie_file:
|
||||||
|
@ -99,10 +114,10 @@ class CookieHandler:
|
||||||
print("cookie: import successful")
|
print("cookie: import successful")
|
||||||
|
|
||||||
def set_cookie(self, cookie):
|
def set_cookie(self, cookie):
|
||||||
"""set cookie str and activate in cofig"""
|
"""set cookie str and activate in config"""
|
||||||
RedisArchivist().set_message("cookie", cookie)
|
RedisArchivist().set_message("cookie", cookie, save=True)
|
||||||
path = ".downloads.cookie_import"
|
path = ".downloads.cookie_import"
|
||||||
RedisArchivist().set_message("config", True, path=path)
|
RedisArchivist().set_message("config", True, path=path, save=True)
|
||||||
self.config["downloads"]["cookie_import"] = True
|
self.config["downloads"]["cookie_import"] = True
|
||||||
print("cookie: activated and stored in Redis")
|
print("cookie: activated and stored in Redis")
|
||||||
|
|
||||||
|
|
|
@ -6,14 +6,13 @@ functionality:
|
||||||
- move to archive
|
- move to archive
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import json
|
|
||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
from home.src.download.queue import PendingList
|
from home.src.download.queue import PendingList
|
||||||
from home.src.download.subscriptions import PlaylistSubscription
|
from home.src.download.subscriptions import PlaylistSubscription
|
||||||
from home.src.download.yt_dlp_base import CookieHandler, YtWrap
|
from home.src.download.yt_dlp_base import YtWrap
|
||||||
from home.src.es.connect import ElasticWrap, IndexPaginate
|
from home.src.es.connect import ElasticWrap, IndexPaginate
|
||||||
from home.src.index.channel import YoutubeChannel
|
from home.src.index.channel import YoutubeChannel
|
||||||
from home.src.index.comments import CommentList
|
from home.src.index.comments import CommentList
|
||||||
|
@ -21,251 +20,102 @@ from home.src.index.playlist import YoutubePlaylist
|
||||||
from home.src.index.video import YoutubeVideo, index_new_video
|
from home.src.index.video import YoutubeVideo, index_new_video
|
||||||
from home.src.index.video_constants import VideoTypeEnum
|
from home.src.index.video_constants import VideoTypeEnum
|
||||||
from home.src.ta.config import AppConfig
|
from home.src.ta.config import AppConfig
|
||||||
from home.src.ta.helper import clean_string, ignore_filelist
|
from home.src.ta.helper import get_channel_overwrites, ignore_filelist
|
||||||
|
from home.src.ta.settings import EnvironmentSettings
|
||||||
from home.src.ta.ta_redis import RedisQueue
|
from home.src.ta.ta_redis import RedisQueue
|
||||||
|
|
||||||
|
|
||||||
class DownloadPostProcess:
|
class DownloaderBase:
|
||||||
"""handle task to run after download queue finishes"""
|
"""base class for shared config"""
|
||||||
|
|
||||||
def __init__(self, download):
|
CACHE_DIR = EnvironmentSettings.CACHE_DIR
|
||||||
self.download = download
|
MEDIA_DIR = EnvironmentSettings.MEDIA_DIR
|
||||||
self.now = int(datetime.now().timestamp())
|
CHANNEL_QUEUE = "download:channel"
|
||||||
self.pending = False
|
PLAYLIST_QUEUE = "download:playlist:full"
|
||||||
|
PLAYLIST_QUICK = "download:playlist:quick"
|
||||||
|
VIDEO_QUEUE = "download:video"
|
||||||
|
|
||||||
def run(self):
|
def __init__(self, task):
|
||||||
"""run all functions"""
|
|
||||||
self.pending = PendingList()
|
|
||||||
self.pending.get_download()
|
|
||||||
self.pending.get_channels()
|
|
||||||
self.pending.get_indexed()
|
|
||||||
self.auto_delete_all()
|
|
||||||
self.auto_delete_overwrites()
|
|
||||||
self.validate_playlists()
|
|
||||||
self.get_comments()
|
|
||||||
|
|
||||||
def auto_delete_all(self):
|
|
||||||
"""handle auto delete"""
|
|
||||||
autodelete_days = self.download.config["downloads"]["autodelete_days"]
|
|
||||||
if not autodelete_days:
|
|
||||||
return
|
|
||||||
|
|
||||||
print(f"auto delete older than {autodelete_days} days")
|
|
||||||
now_lte = self.now - autodelete_days * 24 * 60 * 60
|
|
||||||
data = {
|
|
||||||
"query": {"range": {"player.watched_date": {"lte": now_lte}}},
|
|
||||||
"sort": [{"player.watched_date": {"order": "asc"}}],
|
|
||||||
}
|
|
||||||
self._auto_delete_watched(data)
|
|
||||||
|
|
||||||
def auto_delete_overwrites(self):
|
|
||||||
"""handle per channel auto delete from overwrites"""
|
|
||||||
for channel_id, value in self.pending.channel_overwrites.items():
|
|
||||||
if "autodelete_days" in value:
|
|
||||||
autodelete_days = value.get("autodelete_days")
|
|
||||||
print(f"{channel_id}: delete older than {autodelete_days}d")
|
|
||||||
now_lte = self.now - autodelete_days * 24 * 60 * 60
|
|
||||||
must_list = [
|
|
||||||
{"range": {"player.watched_date": {"lte": now_lte}}},
|
|
||||||
{"term": {"channel.channel_id": {"value": channel_id}}},
|
|
||||||
]
|
|
||||||
data = {
|
|
||||||
"query": {"bool": {"must": must_list}},
|
|
||||||
"sort": [{"player.watched_date": {"order": "desc"}}],
|
|
||||||
}
|
|
||||||
self._auto_delete_watched(data)
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _auto_delete_watched(data):
|
|
||||||
"""delete watched videos after x days"""
|
|
||||||
to_delete = IndexPaginate("ta_video", data).get_results()
|
|
||||||
if not to_delete:
|
|
||||||
return
|
|
||||||
|
|
||||||
for video in to_delete:
|
|
||||||
youtube_id = video["youtube_id"]
|
|
||||||
print(f"{youtube_id}: auto delete video")
|
|
||||||
YoutubeVideo(youtube_id).delete_media_file()
|
|
||||||
|
|
||||||
print("add deleted to ignore list")
|
|
||||||
vids = [{"type": "video", "url": i["youtube_id"]} for i in to_delete]
|
|
||||||
pending = PendingList(youtube_ids=vids)
|
|
||||||
pending.parse_url_list()
|
|
||||||
pending.add_to_pending(status="ignore")
|
|
||||||
|
|
||||||
def validate_playlists(self):
|
|
||||||
"""look for playlist needing to update"""
|
|
||||||
for id_c, channel_id in enumerate(self.download.channels):
|
|
||||||
channel = YoutubeChannel(channel_id, task=self.download.task)
|
|
||||||
overwrites = self.pending.channel_overwrites.get(channel_id, False)
|
|
||||||
if overwrites and overwrites.get("index_playlists"):
|
|
||||||
# validate from remote
|
|
||||||
channel.index_channel_playlists()
|
|
||||||
continue
|
|
||||||
|
|
||||||
# validate from local
|
|
||||||
playlists = channel.get_indexed_playlists(active_only=True)
|
|
||||||
all_channel_playlist = [i["playlist_id"] for i in playlists]
|
|
||||||
self._validate_channel_playlist(all_channel_playlist, id_c)
|
|
||||||
|
|
||||||
def _validate_channel_playlist(self, all_channel_playlist, id_c):
|
|
||||||
"""scan channel for playlist needing update"""
|
|
||||||
all_youtube_ids = [i["youtube_id"] for i in self.pending.all_videos]
|
|
||||||
for id_p, playlist_id in enumerate(all_channel_playlist):
|
|
||||||
playlist = YoutubePlaylist(playlist_id)
|
|
||||||
playlist.all_youtube_ids = all_youtube_ids
|
|
||||||
playlist.build_json(scrape=True)
|
|
||||||
if not playlist.json_data:
|
|
||||||
playlist.deactivate()
|
|
||||||
continue
|
|
||||||
|
|
||||||
playlist.add_vids_to_playlist()
|
|
||||||
playlist.upload_to_es()
|
|
||||||
self._notify_playlist_progress(all_channel_playlist, id_c, id_p)
|
|
||||||
|
|
||||||
def _notify_playlist_progress(self, all_channel_playlist, id_c, id_p):
|
|
||||||
"""notify to UI"""
|
|
||||||
if not self.download.task:
|
|
||||||
return
|
|
||||||
|
|
||||||
total_channel = len(self.download.channels)
|
|
||||||
total_playlist = len(all_channel_playlist)
|
|
||||||
|
|
||||||
message = [
|
|
||||||
f"Post Processing Channels: {id_c}/{total_channel}",
|
|
||||||
f"Validate Playlists {id_p + 1}/{total_playlist}",
|
|
||||||
]
|
|
||||||
progress = (id_c + 1) / total_channel
|
|
||||||
self.download.task.send_progress(message, progress=progress)
|
|
||||||
|
|
||||||
def get_comments(self):
|
|
||||||
"""get comments from youtube"""
|
|
||||||
CommentList(self.download.videos, task=self.download.task).index()
|
|
||||||
|
|
||||||
|
|
||||||
class VideoDownloader:
|
|
||||||
"""
|
|
||||||
handle the video download functionality
|
|
||||||
if not initiated with list, take from queue
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, youtube_id_list=False, task=False):
|
|
||||||
self.obs = False
|
|
||||||
self.video_overwrites = False
|
|
||||||
self.youtube_id_list = youtube_id_list
|
|
||||||
self.task = task
|
self.task = task
|
||||||
self.config = AppConfig().config
|
self.config = AppConfig().config
|
||||||
|
self.channel_overwrites = get_channel_overwrites()
|
||||||
|
self.now = int(datetime.now().timestamp())
|
||||||
|
|
||||||
|
|
||||||
|
class VideoDownloader(DownloaderBase):
|
||||||
|
"""handle the video download functionality"""
|
||||||
|
|
||||||
|
def __init__(self, task=False):
|
||||||
|
super().__init__(task)
|
||||||
|
self.obs = False
|
||||||
self._build_obs()
|
self._build_obs()
|
||||||
self.channels = set()
|
|
||||||
self.videos = set()
|
|
||||||
|
|
||||||
def run_queue(self):
|
def run_queue(self, auto_only=False) -> int:
|
||||||
"""setup download queue in redis loop until no more items"""
|
"""setup download queue in redis loop until no more items"""
|
||||||
self._setup_queue()
|
downloaded = 0
|
||||||
queue = RedisQueue(queue_name="dl_queue")
|
|
||||||
|
|
||||||
limit_queue = self.config["downloads"]["limit_count"]
|
|
||||||
if limit_queue:
|
|
||||||
queue.trim(limit_queue - 1)
|
|
||||||
|
|
||||||
while True:
|
while True:
|
||||||
youtube_data = queue.get_next()
|
video_data = self._get_next(auto_only)
|
||||||
if self.task.is_stopped() or not youtube_data:
|
if self.task.is_stopped() or not video_data:
|
||||||
queue.clear()
|
self._reset_auto()
|
||||||
break
|
break
|
||||||
|
|
||||||
youtube_data = json.loads(youtube_data)
|
youtube_id = video_data["youtube_id"]
|
||||||
youtube_id = youtube_data.get("youtube_id")
|
channel_id = video_data["channel_id"]
|
||||||
|
print(f"{youtube_id}: Downloading video")
|
||||||
|
self._notify(video_data, "Validate download format")
|
||||||
|
|
||||||
tmp_vid_type = youtube_data.get(
|
success = self._dl_single_vid(youtube_id, channel_id)
|
||||||
"vid_type", VideoTypeEnum.VIDEOS.value
|
|
||||||
)
|
|
||||||
video_type = VideoTypeEnum(tmp_vid_type)
|
|
||||||
print(f"{youtube_id}: Downloading type: {video_type}")
|
|
||||||
|
|
||||||
success = self._dl_single_vid(youtube_id)
|
|
||||||
if not success:
|
if not success:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if self.task:
|
self._notify(video_data, "Add video metadata to index", progress=1)
|
||||||
self.task.send_progress(
|
video_type = VideoTypeEnum(video_data["vid_type"])
|
||||||
[
|
vid_dict = index_new_video(youtube_id, video_type=video_type)
|
||||||
f"Processing video {youtube_id}",
|
RedisQueue(self.CHANNEL_QUEUE).add(channel_id)
|
||||||
"Add video metadata to index.",
|
RedisQueue(self.VIDEO_QUEUE).add(youtube_id)
|
||||||
]
|
|
||||||
)
|
|
||||||
|
|
||||||
vid_dict = index_new_video(
|
|
||||||
youtube_id,
|
|
||||||
video_overwrites=self.video_overwrites,
|
|
||||||
video_type=video_type,
|
|
||||||
)
|
|
||||||
self.channels.add(vid_dict["channel"]["channel_id"])
|
|
||||||
self.videos.add(vid_dict["youtube_id"])
|
|
||||||
|
|
||||||
if self.task:
|
|
||||||
self.task.send_progress(
|
|
||||||
[
|
|
||||||
f"Processing video {youtube_id}",
|
|
||||||
"Move downloaded file to archive.",
|
|
||||||
]
|
|
||||||
)
|
|
||||||
|
|
||||||
|
self._notify(video_data, "Move downloaded file to archive")
|
||||||
self.move_to_archive(vid_dict)
|
self.move_to_archive(vid_dict)
|
||||||
|
|
||||||
if queue.has_item():
|
|
||||||
message = "Continue with next video."
|
|
||||||
else:
|
|
||||||
message = "Download queue is finished."
|
|
||||||
|
|
||||||
if self.task:
|
|
||||||
self.task.send_progress([message])
|
|
||||||
|
|
||||||
self._delete_from_pending(youtube_id)
|
self._delete_from_pending(youtube_id)
|
||||||
|
downloaded += 1
|
||||||
|
|
||||||
# post processing
|
# post processing
|
||||||
self._add_subscribed_channels()
|
DownloadPostProcess(self.task).run()
|
||||||
DownloadPostProcess(self).run()
|
|
||||||
|
|
||||||
def _setup_queue(self):
|
return downloaded
|
||||||
"""setup required and validate"""
|
|
||||||
if self.config["downloads"]["cookie_import"]:
|
|
||||||
valid = CookieHandler(self.config).validate()
|
|
||||||
if not valid:
|
|
||||||
return
|
|
||||||
|
|
||||||
pending = PendingList()
|
|
||||||
pending.get_download()
|
|
||||||
pending.get_channels()
|
|
||||||
self.video_overwrites = pending.video_overwrites
|
|
||||||
|
|
||||||
def add_pending(self):
|
|
||||||
"""add pending videos to download queue"""
|
|
||||||
if self.task:
|
|
||||||
self.task.send_progress(["Scanning your download queue."])
|
|
||||||
|
|
||||||
pending = PendingList()
|
|
||||||
pending.get_download()
|
|
||||||
to_add = [
|
|
||||||
json.dumps(
|
|
||||||
{
|
|
||||||
"youtube_id": i["youtube_id"],
|
|
||||||
# Using .value in default val to match what would be
|
|
||||||
# decoded when parsing json if not set
|
|
||||||
"vid_type": i.get("vid_type", VideoTypeEnum.VIDEOS.value),
|
|
||||||
}
|
|
||||||
)
|
|
||||||
for i in pending.all_pending
|
|
||||||
]
|
|
||||||
if not to_add:
|
|
||||||
# there is nothing pending
|
|
||||||
print("download queue is empty")
|
|
||||||
if self.task:
|
|
||||||
self.task.send_progress(["Download queue is empty."])
|
|
||||||
|
|
||||||
|
def _notify(self, video_data, message, progress=False):
|
||||||
|
"""send progress notification to task"""
|
||||||
|
if not self.task:
|
||||||
return
|
return
|
||||||
|
|
||||||
RedisQueue(queue_name="dl_queue").add_list(to_add)
|
typ = VideoTypeEnum(video_data["vid_type"]).value.rstrip("s").title()
|
||||||
|
title = video_data.get("title")
|
||||||
|
self.task.send_progress(
|
||||||
|
[f"Processing {typ}: {title}", message], progress=progress
|
||||||
|
)
|
||||||
|
|
||||||
|
def _get_next(self, auto_only):
|
||||||
|
"""get next item in queue"""
|
||||||
|
must_list = [{"term": {"status": {"value": "pending"}}}]
|
||||||
|
must_not_list = [{"exists": {"field": "message"}}]
|
||||||
|
if auto_only:
|
||||||
|
must_list.append({"term": {"auto_start": {"value": True}}})
|
||||||
|
|
||||||
|
data = {
|
||||||
|
"size": 1,
|
||||||
|
"query": {"bool": {"must": must_list, "must_not": must_not_list}},
|
||||||
|
"sort": [
|
||||||
|
{"auto_start": {"order": "desc"}},
|
||||||
|
{"timestamp": {"order": "asc"}},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
path = "ta_download/_search"
|
||||||
|
response, _ = ElasticWrap(path).get(data=data)
|
||||||
|
if not response["hits"]["hits"]:
|
||||||
|
return False
|
||||||
|
|
||||||
|
return response["hits"]["hits"][0]["_source"]
|
||||||
|
|
||||||
def _progress_hook(self, response):
|
def _progress_hook(self, response):
|
||||||
"""process the progress_hooks from yt_dlp"""
|
"""process the progress_hooks from yt_dlp"""
|
||||||
|
@ -297,10 +147,7 @@ class VideoDownloader:
|
||||||
"""initial obs"""
|
"""initial obs"""
|
||||||
self.obs = {
|
self.obs = {
|
||||||
"merge_output_format": "mp4",
|
"merge_output_format": "mp4",
|
||||||
"outtmpl": (
|
"outtmpl": (self.CACHE_DIR + "/download/%(id)s.mp4"),
|
||||||
self.config["application"]["cache_dir"]
|
|
||||||
+ "/download/%(id)s.mp4"
|
|
||||||
),
|
|
||||||
"progress_hooks": [self._progress_hook],
|
"progress_hooks": [self._progress_hook],
|
||||||
"noprogress": True,
|
"noprogress": True,
|
||||||
"continuedl": True,
|
"continuedl": True,
|
||||||
|
@ -312,6 +159,10 @@ class VideoDownloader:
|
||||||
"""build user customized options"""
|
"""build user customized options"""
|
||||||
if self.config["downloads"]["format"]:
|
if self.config["downloads"]["format"]:
|
||||||
self.obs["format"] = self.config["downloads"]["format"]
|
self.obs["format"] = self.config["downloads"]["format"]
|
||||||
|
if self.config["downloads"]["format_sort"]:
|
||||||
|
format_sort = self.config["downloads"]["format_sort"]
|
||||||
|
format_sort_list = [i.strip() for i in format_sort.split(",")]
|
||||||
|
self.obs["format_sort"] = format_sort_list
|
||||||
if self.config["downloads"]["limit_speed"]:
|
if self.config["downloads"]["limit_speed"]:
|
||||||
self.obs["ratelimit"] = (
|
self.obs["ratelimit"] = (
|
||||||
self.config["downloads"]["limit_speed"] * 1024
|
self.config["downloads"]["limit_speed"] * 1024
|
||||||
|
@ -356,22 +207,17 @@ class VideoDownloader:
|
||||||
|
|
||||||
self.obs["postprocessors"] = postprocessors
|
self.obs["postprocessors"] = postprocessors
|
||||||
|
|
||||||
def get_format_overwrites(self, youtube_id):
|
def _set_overwrites(self, obs: dict, channel_id: str) -> None:
|
||||||
"""get overwrites from single video"""
|
"""add overwrites to obs"""
|
||||||
overwrites = self.video_overwrites.get(youtube_id, False)
|
overwrites = self.channel_overwrites.get(channel_id)
|
||||||
if overwrites:
|
if overwrites and overwrites.get("download_format"):
|
||||||
return overwrites.get("download_format", False)
|
obs["format"] = overwrites.get("download_format")
|
||||||
|
|
||||||
return False
|
def _dl_single_vid(self, youtube_id: str, channel_id: str) -> bool:
|
||||||
|
|
||||||
def _dl_single_vid(self, youtube_id):
|
|
||||||
"""download single video"""
|
"""download single video"""
|
||||||
obs = self.obs.copy()
|
obs = self.obs.copy()
|
||||||
format_overwrite = self.get_format_overwrites(youtube_id)
|
self._set_overwrites(obs, channel_id)
|
||||||
if format_overwrite:
|
dl_cache = os.path.join(self.CACHE_DIR, "download")
|
||||||
obs["format"] = format_overwrite
|
|
||||||
|
|
||||||
dl_cache = self.config["application"]["cache_dir"] + "/download/"
|
|
||||||
|
|
||||||
# check if already in cache to continue from there
|
# check if already in cache to continue from there
|
||||||
all_cached = ignore_filelist(os.listdir(dl_cache))
|
all_cached = ignore_filelist(os.listdir(dl_cache))
|
||||||
|
@ -379,7 +225,9 @@ class VideoDownloader:
|
||||||
if youtube_id in file_name:
|
if youtube_id in file_name:
|
||||||
obs["outtmpl"] = os.path.join(dl_cache, file_name)
|
obs["outtmpl"] = os.path.join(dl_cache, file_name)
|
||||||
|
|
||||||
success = YtWrap(obs, self.config).download(youtube_id)
|
success, message = YtWrap(obs, self.config).download(youtube_id)
|
||||||
|
if not success:
|
||||||
|
self._handle_error(youtube_id, message)
|
||||||
|
|
||||||
if self.obs["writethumbnail"]:
|
if self.obs["writethumbnail"]:
|
||||||
# webp files don't get cleaned up automatically
|
# webp files don't get cleaned up automatically
|
||||||
|
@ -391,29 +239,28 @@ class VideoDownloader:
|
||||||
|
|
||||||
return success
|
return success
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _handle_error(youtube_id, message):
|
||||||
|
"""store error message"""
|
||||||
|
data = {"doc": {"message": message}}
|
||||||
|
_, _ = ElasticWrap(f"ta_download/_update/{youtube_id}").post(data=data)
|
||||||
|
|
||||||
def move_to_archive(self, vid_dict):
|
def move_to_archive(self, vid_dict):
|
||||||
"""move downloaded video from cache to archive"""
|
"""move downloaded video from cache to archive"""
|
||||||
videos = self.config["application"]["videos"]
|
host_uid = EnvironmentSettings.HOST_UID
|
||||||
host_uid = self.config["application"]["HOST_UID"]
|
host_gid = EnvironmentSettings.HOST_GID
|
||||||
host_gid = self.config["application"]["HOST_GID"]
|
# make folder
|
||||||
channel_name = clean_string(vid_dict["channel"]["channel_name"])
|
folder = os.path.join(
|
||||||
if len(channel_name) <= 3:
|
self.MEDIA_DIR, vid_dict["channel"]["channel_id"]
|
||||||
# fall back to channel id
|
)
|
||||||
channel_name = vid_dict["channel"]["channel_id"]
|
if not os.path.exists(folder):
|
||||||
# make archive folder with correct permissions
|
os.makedirs(folder)
|
||||||
new_folder = os.path.join(videos, channel_name)
|
if host_uid and host_gid:
|
||||||
if not os.path.exists(new_folder):
|
os.chown(folder, host_uid, host_gid)
|
||||||
os.makedirs(new_folder)
|
# move media file
|
||||||
if host_uid and host_gid:
|
media_file = vid_dict["youtube_id"] + ".mp4"
|
||||||
os.chown(new_folder, host_uid, host_gid)
|
old_path = os.path.join(self.CACHE_DIR, "download", media_file)
|
||||||
# find real filename
|
new_path = os.path.join(self.MEDIA_DIR, vid_dict["media_url"])
|
||||||
cache_dir = self.config["application"]["cache_dir"]
|
|
||||||
all_cached = ignore_filelist(os.listdir(cache_dir + "/download/"))
|
|
||||||
for file_str in all_cached:
|
|
||||||
if vid_dict["youtube_id"] in file_str:
|
|
||||||
old_file = file_str
|
|
||||||
old_path = os.path.join(cache_dir, "download", old_file)
|
|
||||||
new_path = os.path.join(videos, vid_dict["media_url"])
|
|
||||||
# move media file and fix permission
|
# move media file and fix permission
|
||||||
shutil.move(old_path, new_path, copy_function=shutil.copyfile)
|
shutil.move(old_path, new_path, copy_function=shutil.copyfile)
|
||||||
if host_uid and host_gid:
|
if host_uid and host_gid:
|
||||||
|
@ -422,17 +269,186 @@ class VideoDownloader:
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _delete_from_pending(youtube_id):
|
def _delete_from_pending(youtube_id):
|
||||||
"""delete downloaded video from pending index if its there"""
|
"""delete downloaded video from pending index if its there"""
|
||||||
path = f"ta_download/_doc/{youtube_id}"
|
path = f"ta_download/_doc/{youtube_id}?refresh=true"
|
||||||
_, _ = ElasticWrap(path).delete()
|
_, _ = ElasticWrap(path).delete()
|
||||||
|
|
||||||
def _add_subscribed_channels(self):
|
def _reset_auto(self):
|
||||||
"""add all channels subscribed to refresh"""
|
"""reset autostart to defaults after queue stop"""
|
||||||
all_subscribed = PlaylistSubscription().get_playlists()
|
path = "ta_download/_update_by_query"
|
||||||
if not all_subscribed:
|
data = {
|
||||||
|
"query": {"term": {"auto_start": {"value": True}}},
|
||||||
|
"script": {
|
||||||
|
"source": "ctx._source.auto_start = false",
|
||||||
|
"lang": "painless",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
response, _ = ElasticWrap(path).post(data=data)
|
||||||
|
updated = response.get("updated")
|
||||||
|
if updated:
|
||||||
|
print(f"[download] reset auto start on {updated} videos.")
|
||||||
|
|
||||||
|
|
||||||
|
class DownloadPostProcess(DownloaderBase):
|
||||||
|
"""handle task to run after download queue finishes"""
|
||||||
|
|
||||||
|
def run(self):
|
||||||
|
"""run all functions"""
|
||||||
|
self.auto_delete_all()
|
||||||
|
self.auto_delete_overwrites()
|
||||||
|
self.refresh_playlist()
|
||||||
|
self.match_videos()
|
||||||
|
self.get_comments()
|
||||||
|
|
||||||
|
def auto_delete_all(self):
|
||||||
|
"""handle auto delete"""
|
||||||
|
autodelete_days = self.config["downloads"]["autodelete_days"]
|
||||||
|
if not autodelete_days:
|
||||||
return
|
return
|
||||||
|
|
||||||
channel_ids = [i["playlist_channel_id"] for i in all_subscribed]
|
print(f"auto delete older than {autodelete_days} days")
|
||||||
for channel_id in channel_ids:
|
now_lte = str(self.now - autodelete_days * 24 * 60 * 60)
|
||||||
self.channels.add(channel_id)
|
data = {
|
||||||
|
"query": {"range": {"player.watched_date": {"lte": now_lte}}},
|
||||||
|
"sort": [{"player.watched_date": {"order": "asc"}}],
|
||||||
|
}
|
||||||
|
self._auto_delete_watched(data)
|
||||||
|
|
||||||
return
|
def auto_delete_overwrites(self):
|
||||||
|
"""handle per channel auto delete from overwrites"""
|
||||||
|
for channel_id, value in self.channel_overwrites.items():
|
||||||
|
if "autodelete_days" in value:
|
||||||
|
autodelete_days = value.get("autodelete_days")
|
||||||
|
print(f"{channel_id}: delete older than {autodelete_days}d")
|
||||||
|
now_lte = str(self.now - autodelete_days * 24 * 60 * 60)
|
||||||
|
must_list = [
|
||||||
|
{"range": {"player.watched_date": {"lte": now_lte}}},
|
||||||
|
{"term": {"channel.channel_id": {"value": channel_id}}},
|
||||||
|
]
|
||||||
|
data = {
|
||||||
|
"query": {"bool": {"must": must_list}},
|
||||||
|
"sort": [{"player.watched_date": {"order": "desc"}}],
|
||||||
|
}
|
||||||
|
self._auto_delete_watched(data)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _auto_delete_watched(data):
|
||||||
|
"""delete watched videos after x days"""
|
||||||
|
to_delete = IndexPaginate("ta_video", data).get_results()
|
||||||
|
if not to_delete:
|
||||||
|
return
|
||||||
|
|
||||||
|
for video in to_delete:
|
||||||
|
youtube_id = video["youtube_id"]
|
||||||
|
print(f"{youtube_id}: auto delete video")
|
||||||
|
YoutubeVideo(youtube_id).delete_media_file()
|
||||||
|
|
||||||
|
print("add deleted to ignore list")
|
||||||
|
vids = [{"type": "video", "url": i["youtube_id"]} for i in to_delete]
|
||||||
|
pending = PendingList(youtube_ids=vids)
|
||||||
|
pending.parse_url_list()
|
||||||
|
_ = pending.add_to_pending(status="ignore")
|
||||||
|
|
||||||
|
def refresh_playlist(self) -> None:
|
||||||
|
"""match videos with playlists"""
|
||||||
|
self.add_playlists_to_refresh()
|
||||||
|
|
||||||
|
queue = RedisQueue(self.PLAYLIST_QUEUE)
|
||||||
|
while True:
|
||||||
|
total = queue.max_score()
|
||||||
|
playlist_id, idx = queue.get_next()
|
||||||
|
if not playlist_id or not idx or not total:
|
||||||
|
break
|
||||||
|
|
||||||
|
playlist = YoutubePlaylist(playlist_id)
|
||||||
|
playlist.update_playlist(skip_on_empty=True)
|
||||||
|
|
||||||
|
if not self.task:
|
||||||
|
continue
|
||||||
|
|
||||||
|
channel_name = playlist.json_data["playlist_channel"]
|
||||||
|
playlist_title = playlist.json_data["playlist_name"]
|
||||||
|
message = [
|
||||||
|
f"Post Processing Playlists for: {channel_name}",
|
||||||
|
f"{playlist_title} [{idx}/{total}]",
|
||||||
|
]
|
||||||
|
progress = idx / total
|
||||||
|
self.task.send_progress(message, progress=progress)
|
||||||
|
|
||||||
|
def add_playlists_to_refresh(self) -> None:
|
||||||
|
"""add playlists to refresh"""
|
||||||
|
if self.task:
|
||||||
|
message = ["Post Processing Playlists", "Scanning for Playlists"]
|
||||||
|
self.task.send_progress(message)
|
||||||
|
|
||||||
|
self._add_playlist_sub()
|
||||||
|
self._add_channel_playlists()
|
||||||
|
self._add_video_playlists()
|
||||||
|
|
||||||
|
def _add_playlist_sub(self):
|
||||||
|
"""add subscribed playlists to refresh"""
|
||||||
|
subs = PlaylistSubscription().get_playlists()
|
||||||
|
to_add = [i["playlist_id"] for i in subs]
|
||||||
|
RedisQueue(self.PLAYLIST_QUEUE).add_list(to_add)
|
||||||
|
|
||||||
|
def _add_channel_playlists(self):
|
||||||
|
"""add playlists from channels to refresh"""
|
||||||
|
queue = RedisQueue(self.CHANNEL_QUEUE)
|
||||||
|
while True:
|
||||||
|
channel_id, _ = queue.get_next()
|
||||||
|
if not channel_id:
|
||||||
|
break
|
||||||
|
|
||||||
|
channel = YoutubeChannel(channel_id)
|
||||||
|
channel.get_from_es()
|
||||||
|
overwrites = channel.get_overwrites()
|
||||||
|
if "index_playlists" in overwrites:
|
||||||
|
channel.get_all_playlists()
|
||||||
|
to_add = [i[0] for i in channel.all_playlists]
|
||||||
|
RedisQueue(self.PLAYLIST_QUEUE).add_list(to_add)
|
||||||
|
|
||||||
|
def _add_video_playlists(self):
|
||||||
|
"""add other playlists for quick sync"""
|
||||||
|
all_playlists = RedisQueue(self.PLAYLIST_QUEUE).get_all()
|
||||||
|
must_not = [{"terms": {"playlist_id": all_playlists}}]
|
||||||
|
video_ids = RedisQueue(self.VIDEO_QUEUE).get_all()
|
||||||
|
must = [{"terms": {"playlist_entries.youtube_id": video_ids}}]
|
||||||
|
data = {
|
||||||
|
"query": {"bool": {"must_not": must_not, "must": must}},
|
||||||
|
"_source": ["playlist_id"],
|
||||||
|
}
|
||||||
|
playlists = IndexPaginate("ta_playlist", data).get_results()
|
||||||
|
to_add = [i["playlist_id"] for i in playlists]
|
||||||
|
RedisQueue(self.PLAYLIST_QUICK).add_list(to_add)
|
||||||
|
|
||||||
|
def match_videos(self) -> None:
|
||||||
|
"""scan rest of indexed playlists to match videos"""
|
||||||
|
queue = RedisQueue(self.PLAYLIST_QUICK)
|
||||||
|
while True:
|
||||||
|
total = queue.max_score()
|
||||||
|
playlist_id, idx = queue.get_next()
|
||||||
|
if not playlist_id or not idx or not total:
|
||||||
|
break
|
||||||
|
|
||||||
|
playlist = YoutubePlaylist(playlist_id)
|
||||||
|
playlist.get_from_es()
|
||||||
|
playlist.add_vids_to_playlist()
|
||||||
|
playlist.remove_vids_from_playlist()
|
||||||
|
|
||||||
|
if not self.task:
|
||||||
|
continue
|
||||||
|
|
||||||
|
message = [
|
||||||
|
"Post Processing Playlists.",
|
||||||
|
f"Validate Playlists: - {idx}/{total}",
|
||||||
|
]
|
||||||
|
progress = idx / total
|
||||||
|
self.task.send_progress(message, progress=progress)
|
||||||
|
|
||||||
|
def get_comments(self):
|
||||||
|
"""get comments from youtube"""
|
||||||
|
video_queue = RedisQueue(self.VIDEO_QUEUE)
|
||||||
|
comment_list = CommentList(task=self.task)
|
||||||
|
comment_list.add(video_ids=video_queue.get_all())
|
||||||
|
|
||||||
|
video_queue.clear()
|
||||||
|
comment_list.index()
|
||||||
|
|
|
@ -10,17 +10,22 @@ import os
|
||||||
import zipfile
|
import zipfile
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
|
from home.models import CustomPeriodicTask
|
||||||
from home.src.es.connect import ElasticWrap, IndexPaginate
|
from home.src.es.connect import ElasticWrap, IndexPaginate
|
||||||
from home.src.ta.config import AppConfig
|
from home.src.ta.config import AppConfig
|
||||||
from home.src.ta.helper import get_mapping, ignore_filelist
|
from home.src.ta.helper import get_mapping, ignore_filelist
|
||||||
|
from home.src.ta.settings import EnvironmentSettings
|
||||||
|
|
||||||
|
|
||||||
class ElasticBackup:
|
class ElasticBackup:
|
||||||
"""dump index to nd-json files for later bulk import"""
|
"""dump index to nd-json files for later bulk import"""
|
||||||
|
|
||||||
|
INDEX_SPLIT = ["comment"]
|
||||||
|
CACHE_DIR = EnvironmentSettings.CACHE_DIR
|
||||||
|
BACKUP_DIR = os.path.join(CACHE_DIR, "backup")
|
||||||
|
|
||||||
def __init__(self, reason=False, task=False):
|
def __init__(self, reason=False, task=False):
|
||||||
self.config = AppConfig().config
|
self.config = AppConfig().config
|
||||||
self.cache_dir = self.config["application"]["cache_dir"]
|
|
||||||
self.timestamp = datetime.now().strftime("%Y%m%d")
|
self.timestamp = datetime.now().strftime("%Y%m%d")
|
||||||
self.index_config = get_mapping()
|
self.index_config = get_mapping()
|
||||||
self.reason = reason
|
self.reason = reason
|
||||||
|
@ -32,7 +37,8 @@ class ElasticBackup:
|
||||||
if not self.reason:
|
if not self.reason:
|
||||||
raise ValueError("missing backup reason in ElasticBackup")
|
raise ValueError("missing backup reason in ElasticBackup")
|
||||||
|
|
||||||
self.task.send_progress(["Scanning your index."])
|
if self.task:
|
||||||
|
self.task.send_progress(["Scanning your index."])
|
||||||
for index in self.index_config:
|
for index in self.index_config:
|
||||||
index_name = index["index_name"]
|
index_name = index["index_name"]
|
||||||
print(f"backup: export in progress for {index_name}")
|
print(f"backup: export in progress for {index_name}")
|
||||||
|
@ -42,21 +48,26 @@ class ElasticBackup:
|
||||||
|
|
||||||
self.backup_index(index_name)
|
self.backup_index(index_name)
|
||||||
|
|
||||||
self.task.send_progress(["Compress files to zip archive."])
|
if self.task:
|
||||||
|
self.task.send_progress(["Compress files to zip archive."])
|
||||||
self.zip_it()
|
self.zip_it()
|
||||||
if self.reason == "auto":
|
if self.reason == "auto":
|
||||||
self.rotate_backup()
|
self.rotate_backup()
|
||||||
|
|
||||||
def backup_index(self, index_name):
|
def backup_index(self, index_name):
|
||||||
"""export all documents of a single index"""
|
"""export all documents of a single index"""
|
||||||
paginate = IndexPaginate(
|
paginate_kwargs = {
|
||||||
f"ta_{index_name}",
|
"data": {"query": {"match_all": {}}},
|
||||||
data={"query": {"match_all": {}}},
|
"keep_source": True,
|
||||||
keep_source=True,
|
"callback": BackupCallback,
|
||||||
callback=BackupCallback,
|
"task": self.task,
|
||||||
task=self.task,
|
"total": self._get_total(index_name),
|
||||||
total=self._get_total(index_name),
|
}
|
||||||
)
|
|
||||||
|
if index_name in self.INDEX_SPLIT:
|
||||||
|
paginate_kwargs.update({"size": 200})
|
||||||
|
|
||||||
|
paginate = IndexPaginate(f"ta_{index_name}", **paginate_kwargs)
|
||||||
_ = paginate.get_results()
|
_ = paginate.get_results()
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
@ -70,14 +81,13 @@ class ElasticBackup:
|
||||||
def zip_it(self):
|
def zip_it(self):
|
||||||
"""pack it up into single zip file"""
|
"""pack it up into single zip file"""
|
||||||
file_name = f"ta_backup-{self.timestamp}-{self.reason}.zip"
|
file_name = f"ta_backup-{self.timestamp}-{self.reason}.zip"
|
||||||
folder = os.path.join(self.cache_dir, "backup")
|
|
||||||
|
|
||||||
to_backup = []
|
to_backup = []
|
||||||
for file in os.listdir(folder):
|
for file in os.listdir(self.BACKUP_DIR):
|
||||||
if file.endswith(".json"):
|
if file.endswith(".json"):
|
||||||
to_backup.append(os.path.join(folder, file))
|
to_backup.append(os.path.join(self.BACKUP_DIR, file))
|
||||||
|
|
||||||
backup_file = os.path.join(folder, file_name)
|
backup_file = os.path.join(self.BACKUP_DIR, file_name)
|
||||||
|
|
||||||
comp = zipfile.ZIP_DEFLATED
|
comp = zipfile.ZIP_DEFLATED
|
||||||
with zipfile.ZipFile(backup_file, "w", compression=comp) as zip_f:
|
with zipfile.ZipFile(backup_file, "w", compression=comp) as zip_f:
|
||||||
|
@ -90,7 +100,7 @@ class ElasticBackup:
|
||||||
|
|
||||||
def post_bulk_restore(self, file_name):
|
def post_bulk_restore(self, file_name):
|
||||||
"""send bulk to es"""
|
"""send bulk to es"""
|
||||||
file_path = os.path.join(self.cache_dir, file_name)
|
file_path = os.path.join(self.CACHE_DIR, file_name)
|
||||||
with open(file_path, "r", encoding="utf-8") as f:
|
with open(file_path, "r", encoding="utf-8") as f:
|
||||||
data = f.read()
|
data = f.read()
|
||||||
|
|
||||||
|
@ -101,9 +111,7 @@ class ElasticBackup:
|
||||||
|
|
||||||
def get_all_backup_files(self):
|
def get_all_backup_files(self):
|
||||||
"""build all available backup files for view"""
|
"""build all available backup files for view"""
|
||||||
backup_dir = os.path.join(self.cache_dir, "backup")
|
all_backup_files = ignore_filelist(os.listdir(self.BACKUP_DIR))
|
||||||
backup_files = os.listdir(backup_dir)
|
|
||||||
all_backup_files = ignore_filelist(backup_files)
|
|
||||||
all_available_backups = [
|
all_available_backups = [
|
||||||
i
|
i
|
||||||
for i in all_backup_files
|
for i in all_backup_files
|
||||||
|
@ -112,24 +120,36 @@ class ElasticBackup:
|
||||||
all_available_backups.sort(reverse=True)
|
all_available_backups.sort(reverse=True)
|
||||||
|
|
||||||
backup_dicts = []
|
backup_dicts = []
|
||||||
for backup_file in all_available_backups:
|
for filename in all_available_backups:
|
||||||
file_split = backup_file.split("-")
|
data = self.build_backup_file_data(filename)
|
||||||
if len(file_split) == 2:
|
backup_dicts.append(data)
|
||||||
timestamp = file_split[1].strip(".zip")
|
|
||||||
reason = False
|
|
||||||
elif len(file_split) == 3:
|
|
||||||
timestamp = file_split[1]
|
|
||||||
reason = file_split[2].strip(".zip")
|
|
||||||
|
|
||||||
to_add = {
|
|
||||||
"filename": backup_file,
|
|
||||||
"timestamp": timestamp,
|
|
||||||
"reason": reason,
|
|
||||||
}
|
|
||||||
backup_dicts.append(to_add)
|
|
||||||
|
|
||||||
return backup_dicts
|
return backup_dicts
|
||||||
|
|
||||||
|
def build_backup_file_data(self, filename):
|
||||||
|
"""build metadata of single backup file"""
|
||||||
|
file_path = os.path.join(self.BACKUP_DIR, filename)
|
||||||
|
if not os.path.exists(file_path):
|
||||||
|
return False
|
||||||
|
|
||||||
|
file_split = filename.split("-")
|
||||||
|
if len(file_split) == 2:
|
||||||
|
timestamp = file_split[1].strip(".zip")
|
||||||
|
reason = False
|
||||||
|
elif len(file_split) == 3:
|
||||||
|
timestamp = file_split[1]
|
||||||
|
reason = file_split[2].strip(".zip")
|
||||||
|
|
||||||
|
data = {
|
||||||
|
"filename": filename,
|
||||||
|
"file_path": file_path,
|
||||||
|
"file_size": os.path.getsize(file_path),
|
||||||
|
"timestamp": timestamp,
|
||||||
|
"reason": reason,
|
||||||
|
}
|
||||||
|
|
||||||
|
return data
|
||||||
|
|
||||||
def restore(self, filename):
|
def restore(self, filename):
|
||||||
"""
|
"""
|
||||||
restore from backup zip file
|
restore from backup zip file
|
||||||
|
@ -140,22 +160,19 @@ class ElasticBackup:
|
||||||
|
|
||||||
def _unpack_zip_backup(self, filename):
|
def _unpack_zip_backup(self, filename):
|
||||||
"""extract backup zip and return filelist"""
|
"""extract backup zip and return filelist"""
|
||||||
backup_dir = os.path.join(self.cache_dir, "backup")
|
file_path = os.path.join(self.BACKUP_DIR, filename)
|
||||||
file_path = os.path.join(backup_dir, filename)
|
|
||||||
|
|
||||||
with zipfile.ZipFile(file_path, "r") as z:
|
with zipfile.ZipFile(file_path, "r") as z:
|
||||||
zip_content = z.namelist()
|
zip_content = z.namelist()
|
||||||
z.extractall(backup_dir)
|
z.extractall(self.BACKUP_DIR)
|
||||||
|
|
||||||
return zip_content
|
return zip_content
|
||||||
|
|
||||||
def _restore_json_files(self, zip_content):
|
def _restore_json_files(self, zip_content):
|
||||||
"""go through the unpacked files and restore"""
|
"""go through the unpacked files and restore"""
|
||||||
backup_dir = os.path.join(self.cache_dir, "backup")
|
|
||||||
|
|
||||||
for idx, json_f in enumerate(zip_content):
|
for idx, json_f in enumerate(zip_content):
|
||||||
self._notify_restore(idx, json_f, len(zip_content))
|
self._notify_restore(idx, json_f, len(zip_content))
|
||||||
file_name = os.path.join(backup_dir, json_f)
|
file_name = os.path.join(self.BACKUP_DIR, json_f)
|
||||||
|
|
||||||
if not json_f.startswith("es_") or not json_f.endswith(".json"):
|
if not json_f.startswith("es_") or not json_f.endswith(".json"):
|
||||||
os.remove(file_name)
|
os.remove(file_name)
|
||||||
|
@ -181,7 +198,12 @@ class ElasticBackup:
|
||||||
|
|
||||||
def rotate_backup(self):
|
def rotate_backup(self):
|
||||||
"""delete old backups if needed"""
|
"""delete old backups if needed"""
|
||||||
rotate = self.config["scheduler"]["run_backup_rotate"]
|
try:
|
||||||
|
task = CustomPeriodicTask.objects.get(name="run_backup")
|
||||||
|
except CustomPeriodicTask.DoesNotExist:
|
||||||
|
return
|
||||||
|
|
||||||
|
rotate = task.task_config.get("rotate")
|
||||||
if not rotate:
|
if not rotate:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
@ -192,22 +214,32 @@ class ElasticBackup:
|
||||||
print("no backup files to rotate")
|
print("no backup files to rotate")
|
||||||
return
|
return
|
||||||
|
|
||||||
backup_dir = os.path.join(self.cache_dir, "backup")
|
|
||||||
|
|
||||||
all_to_delete = auto[rotate:]
|
all_to_delete = auto[rotate:]
|
||||||
for to_delete in all_to_delete:
|
for to_delete in all_to_delete:
|
||||||
file_path = os.path.join(backup_dir, to_delete["filename"])
|
self.delete_file(to_delete["filename"])
|
||||||
print(f"remove old backup file: {file_path}")
|
|
||||||
os.remove(file_path)
|
def delete_file(self, filename):
|
||||||
|
"""delete backup file"""
|
||||||
|
file_path = os.path.join(self.BACKUP_DIR, filename)
|
||||||
|
if not os.path.exists(file_path):
|
||||||
|
print(f"backup file not found: {filename}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
print(f"remove old backup file: {file_path}")
|
||||||
|
os.remove(file_path)
|
||||||
|
|
||||||
|
return file_path
|
||||||
|
|
||||||
|
|
||||||
class BackupCallback:
|
class BackupCallback:
|
||||||
"""handle backup ndjson writer as callback for IndexPaginate"""
|
"""handle backup ndjson writer as callback for IndexPaginate"""
|
||||||
|
|
||||||
def __init__(self, source, index_name):
|
def __init__(self, source, index_name, counter=0):
|
||||||
self.source = source
|
self.source = source
|
||||||
self.index_name = index_name
|
self.index_name = index_name
|
||||||
|
self.counter = counter
|
||||||
self.timestamp = datetime.now().strftime("%Y%m%d")
|
self.timestamp = datetime.now().strftime("%Y%m%d")
|
||||||
|
self.cache_dir = EnvironmentSettings.CACHE_DIR
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
"""run the junk task"""
|
"""run the junk task"""
|
||||||
|
@ -234,8 +266,8 @@ class BackupCallback:
|
||||||
|
|
||||||
def _write_es_json(self, file_content):
|
def _write_es_json(self, file_content):
|
||||||
"""write nd-json file for es _bulk API to disk"""
|
"""write nd-json file for es _bulk API to disk"""
|
||||||
cache_dir = AppConfig().config["application"]["cache_dir"]
|
index = self.index_name.lstrip("ta_")
|
||||||
file_name = f"es_{self.index_name.lstrip('ta_')}-{self.timestamp}.json"
|
file_name = f"es_{index}-{self.timestamp}-{self.counter}.json"
|
||||||
file_path = os.path.join(cache_dir, "backup", file_name)
|
file_path = os.path.join(self.cache_dir, "backup", file_name)
|
||||||
with open(file_path, "a+", encoding="utf-8") as f:
|
with open(file_path, "a+", encoding="utf-8") as f:
|
||||||
f.write(file_content)
|
f.write(file_content)
|
||||||
|
|
|
@ -3,12 +3,15 @@ functionality:
|
||||||
- wrapper around requests to call elastic search
|
- wrapper around requests to call elastic search
|
||||||
- reusable search_after to extract total index
|
- reusable search_after to extract total index
|
||||||
"""
|
"""
|
||||||
|
|
||||||
# pylint: disable=missing-timeout
|
# pylint: disable=missing-timeout
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
from home.src.ta.config import AppConfig
|
import urllib3
|
||||||
|
from home.src.ta.settings import EnvironmentSettings
|
||||||
|
|
||||||
|
|
||||||
class ElasticWrap:
|
class ElasticWrap:
|
||||||
|
@ -16,61 +19,94 @@ class ElasticWrap:
|
||||||
returns response json and status code tuple
|
returns response json and status code tuple
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, path, config=False):
|
def __init__(self, path: str):
|
||||||
self.url = False
|
self.url: str = f"{EnvironmentSettings.ES_URL}/{path}"
|
||||||
self.auth = False
|
self.auth: tuple[str, str] = (
|
||||||
self.path = path
|
EnvironmentSettings.ES_USER,
|
||||||
self.config = config
|
EnvironmentSettings.ES_PASS,
|
||||||
self._get_config()
|
)
|
||||||
|
|
||||||
def _get_config(self):
|
if EnvironmentSettings.ES_DISABLE_VERIFY_SSL:
|
||||||
"""add config if not passed"""
|
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||||||
if not self.config:
|
|
||||||
self.config = AppConfig().config
|
|
||||||
|
|
||||||
es_url = self.config["application"]["es_url"]
|
def get(
|
||||||
self.auth = self.config["application"]["es_auth"]
|
self,
|
||||||
self.url = f"{es_url}/{self.path}"
|
data: bool | dict = False,
|
||||||
|
timeout: int = 10,
|
||||||
def get(self, data=False, timeout=10, print_error=True):
|
print_error: bool = True,
|
||||||
|
) -> tuple[dict, int]:
|
||||||
"""get data from es"""
|
"""get data from es"""
|
||||||
|
|
||||||
|
kwargs: dict[str, Any] = {
|
||||||
|
"auth": self.auth,
|
||||||
|
"timeout": timeout,
|
||||||
|
}
|
||||||
|
|
||||||
|
if EnvironmentSettings.ES_DISABLE_VERIFY_SSL:
|
||||||
|
kwargs["verify"] = False
|
||||||
|
|
||||||
if data:
|
if data:
|
||||||
response = requests.get(
|
kwargs["json"] = data
|
||||||
self.url, json=data, auth=self.auth, timeout=timeout
|
|
||||||
)
|
response = requests.get(self.url, **kwargs)
|
||||||
else:
|
|
||||||
response = requests.get(self.url, auth=self.auth, timeout=timeout)
|
|
||||||
if print_error and not response.ok:
|
if print_error and not response.ok:
|
||||||
print(response.text)
|
print(response.text)
|
||||||
|
|
||||||
return response.json(), response.status_code
|
return response.json(), response.status_code
|
||||||
|
|
||||||
def post(self, data=False, ndjson=False):
|
def post(
|
||||||
|
self, data: bool | dict = False, ndjson: bool = False
|
||||||
|
) -> tuple[dict, int]:
|
||||||
"""post data to es"""
|
"""post data to es"""
|
||||||
if ndjson:
|
|
||||||
headers = {"Content-type": "application/x-ndjson"}
|
|
||||||
payload = data
|
|
||||||
else:
|
|
||||||
headers = {"Content-type": "application/json"}
|
|
||||||
payload = json.dumps(data)
|
|
||||||
|
|
||||||
if data:
|
kwargs: dict[str, Any] = {"auth": self.auth}
|
||||||
response = requests.post(
|
|
||||||
self.url, data=payload, headers=headers, auth=self.auth
|
if ndjson and data:
|
||||||
|
kwargs.update(
|
||||||
|
{
|
||||||
|
"headers": {"Content-type": "application/x-ndjson"},
|
||||||
|
"data": data,
|
||||||
|
}
|
||||||
)
|
)
|
||||||
else:
|
elif data:
|
||||||
response = requests.post(self.url, headers=headers, auth=self.auth)
|
kwargs.update(
|
||||||
|
{
|
||||||
|
"headers": {"Content-type": "application/json"},
|
||||||
|
"data": json.dumps(data),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
if EnvironmentSettings.ES_DISABLE_VERIFY_SSL:
|
||||||
|
kwargs["verify"] = False
|
||||||
|
|
||||||
|
response = requests.post(self.url, **kwargs)
|
||||||
|
|
||||||
if not response.ok:
|
if not response.ok:
|
||||||
print(response.text)
|
print(response.text)
|
||||||
|
|
||||||
return response.json(), response.status_code
|
return response.json(), response.status_code
|
||||||
|
|
||||||
def put(self, data, refresh=False):
|
def put(
|
||||||
|
self,
|
||||||
|
data: bool | dict = False,
|
||||||
|
refresh: bool = False,
|
||||||
|
) -> tuple[dict, Any]:
|
||||||
"""put data to es"""
|
"""put data to es"""
|
||||||
|
|
||||||
if refresh:
|
if refresh:
|
||||||
self.url = f"{self.url}/?refresh=true"
|
self.url = f"{self.url}/?refresh=true"
|
||||||
response = requests.put(f"{self.url}", json=data, auth=self.auth)
|
|
||||||
|
kwargs: dict[str, Any] = {
|
||||||
|
"json": data,
|
||||||
|
"auth": self.auth,
|
||||||
|
}
|
||||||
|
|
||||||
|
if EnvironmentSettings.ES_DISABLE_VERIFY_SSL:
|
||||||
|
kwargs["verify"] = False
|
||||||
|
|
||||||
|
response = requests.put(self.url, **kwargs)
|
||||||
|
|
||||||
if not response.ok:
|
if not response.ok:
|
||||||
print(response.text)
|
print(response.text)
|
||||||
print(data)
|
print(data)
|
||||||
|
@ -78,14 +114,25 @@ class ElasticWrap:
|
||||||
|
|
||||||
return response.json(), response.status_code
|
return response.json(), response.status_code
|
||||||
|
|
||||||
def delete(self, data=False, refresh=False):
|
def delete(
|
||||||
|
self,
|
||||||
|
data: bool | dict = False,
|
||||||
|
refresh: bool = False,
|
||||||
|
) -> tuple[dict, Any]:
|
||||||
"""delete document from es"""
|
"""delete document from es"""
|
||||||
|
|
||||||
if refresh:
|
if refresh:
|
||||||
self.url = f"{self.url}/?refresh=true"
|
self.url = f"{self.url}/?refresh=true"
|
||||||
|
|
||||||
|
kwargs: dict[str, Any] = {"auth": self.auth}
|
||||||
|
|
||||||
if data:
|
if data:
|
||||||
response = requests.delete(self.url, json=data, auth=self.auth)
|
kwargs["json"] = data
|
||||||
else:
|
|
||||||
response = requests.delete(self.url, auth=self.auth)
|
if EnvironmentSettings.ES_DISABLE_VERIFY_SSL:
|
||||||
|
kwargs["verify"] = False
|
||||||
|
|
||||||
|
response = requests.delete(self.url, **kwargs)
|
||||||
|
|
||||||
if not response.ok:
|
if not response.ok:
|
||||||
print(response.text)
|
print(response.text)
|
||||||
|
@ -127,6 +174,12 @@ class IndexPaginate:
|
||||||
|
|
||||||
def validate_data(self):
|
def validate_data(self):
|
||||||
"""add pit and size to data"""
|
"""add pit and size to data"""
|
||||||
|
if not self.data:
|
||||||
|
self.data = {}
|
||||||
|
|
||||||
|
if "query" not in self.data.keys():
|
||||||
|
self.data.update({"query": {"match_all": {}}})
|
||||||
|
|
||||||
if "sort" not in self.data.keys():
|
if "sort" not in self.data.keys():
|
||||||
self.data.update({"sort": [{"_doc": {"order": "desc"}}]})
|
self.data.update({"sort": [{"_doc": {"order": "desc"}}]})
|
||||||
|
|
||||||
|
@ -150,7 +203,9 @@ class IndexPaginate:
|
||||||
all_results.append(hit["_source"])
|
all_results.append(hit["_source"])
|
||||||
|
|
||||||
if self.kwargs.get("callback"):
|
if self.kwargs.get("callback"):
|
||||||
self.kwargs.get("callback")(all_hits, self.index_name).run()
|
self.kwargs.get("callback")(
|
||||||
|
all_hits, self.index_name, counter=counter
|
||||||
|
).run()
|
||||||
|
|
||||||
if self.kwargs.get("task"):
|
if self.kwargs.get("task"):
|
||||||
print(f"{self.index_name}: processing page {counter}")
|
print(f"{self.index_name}: processing page {counter}")
|
||||||
|
|
|
@ -1,5 +1,17 @@
|
||||||
{
|
{
|
||||||
"index_config": [{
|
"index_config": [{
|
||||||
|
"index_name": "config",
|
||||||
|
"expected_map": {
|
||||||
|
"config": {
|
||||||
|
"type": "object",
|
||||||
|
"enabled": false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"expected_set": {
|
||||||
|
"number_of_replicas": "0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
"index_name": "channel",
|
"index_name": "channel",
|
||||||
"expected_map": {
|
"expected_map": {
|
||||||
"channel_id": {
|
"channel_id": {
|
||||||
|
@ -37,7 +49,18 @@
|
||||||
"type": "text"
|
"type": "text"
|
||||||
},
|
},
|
||||||
"channel_last_refresh": {
|
"channel_last_refresh": {
|
||||||
"type": "date"
|
"type": "date",
|
||||||
|
"format": "epoch_second"
|
||||||
|
},
|
||||||
|
"channel_tags": {
|
||||||
|
"type": "text",
|
||||||
|
"analyzer": "english",
|
||||||
|
"fields": {
|
||||||
|
"keyword": {
|
||||||
|
"type": "keyword",
|
||||||
|
"ignore_above": 256
|
||||||
|
}
|
||||||
|
}
|
||||||
},
|
},
|
||||||
"channel_overwrites": {
|
"channel_overwrites": {
|
||||||
"properties": {
|
"properties": {
|
||||||
|
@ -51,7 +74,7 @@
|
||||||
"type": "boolean"
|
"type": "boolean"
|
||||||
},
|
},
|
||||||
"integrate_sponsorblock": {
|
"integrate_sponsorblock": {
|
||||||
"type" : "boolean"
|
"type": "boolean"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -80,7 +103,8 @@
|
||||||
"index": false
|
"index": false
|
||||||
},
|
},
|
||||||
"date_downloaded": {
|
"date_downloaded": {
|
||||||
"type": "date"
|
"type": "date",
|
||||||
|
"format": "epoch_second"
|
||||||
},
|
},
|
||||||
"channel": {
|
"channel": {
|
||||||
"properties": {
|
"properties": {
|
||||||
|
@ -119,7 +143,18 @@
|
||||||
"type": "text"
|
"type": "text"
|
||||||
},
|
},
|
||||||
"channel_last_refresh": {
|
"channel_last_refresh": {
|
||||||
"type": "date"
|
"type": "date",
|
||||||
|
"format": "epoch_second"
|
||||||
|
},
|
||||||
|
"channel_tags": {
|
||||||
|
"type": "text",
|
||||||
|
"analyzer": "english",
|
||||||
|
"fields": {
|
||||||
|
"keyword": {
|
||||||
|
"type": "keyword",
|
||||||
|
"ignore_above": 256
|
||||||
|
}
|
||||||
|
}
|
||||||
},
|
},
|
||||||
"channel_overwrites": {
|
"channel_overwrites": {
|
||||||
"properties": {
|
"properties": {
|
||||||
|
@ -133,7 +168,7 @@
|
||||||
"type": "boolean"
|
"type": "boolean"
|
||||||
},
|
},
|
||||||
"integrate_sponsorblock": {
|
"integrate_sponsorblock": {
|
||||||
"type" : "boolean"
|
"type": "boolean"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -146,6 +181,9 @@
|
||||||
"type": "keyword",
|
"type": "keyword",
|
||||||
"index": false
|
"index": false
|
||||||
},
|
},
|
||||||
|
"media_size": {
|
||||||
|
"type": "long"
|
||||||
|
},
|
||||||
"tags": {
|
"tags": {
|
||||||
"type": "text",
|
"type": "text",
|
||||||
"analyzer": "english",
|
"analyzer": "english",
|
||||||
|
@ -173,7 +211,8 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"vid_last_refresh": {
|
"vid_last_refresh": {
|
||||||
"type": "date"
|
"type": "date",
|
||||||
|
"format": "epoch_second"
|
||||||
},
|
},
|
||||||
"youtube_id": {
|
"youtube_id": {
|
||||||
"type": "keyword"
|
"type": "keyword"
|
||||||
|
@ -197,19 +236,37 @@
|
||||||
"comment_count": {
|
"comment_count": {
|
||||||
"type": "long"
|
"type": "long"
|
||||||
},
|
},
|
||||||
"stats" : {
|
"stats": {
|
||||||
"properties" : {
|
"properties": {
|
||||||
"average_rating" : {
|
"average_rating": {
|
||||||
"type" : "float"
|
"type": "float"
|
||||||
},
|
},
|
||||||
"dislike_count" : {
|
"dislike_count": {
|
||||||
"type" : "long"
|
"type": "long"
|
||||||
},
|
},
|
||||||
"like_count" : {
|
"like_count": {
|
||||||
"type" : "long"
|
"type": "long"
|
||||||
},
|
},
|
||||||
"view_count" : {
|
"view_count": {
|
||||||
"type" : "long"
|
"type": "long"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"player": {
|
||||||
|
"properties": {
|
||||||
|
"duration": {
|
||||||
|
"type": "long"
|
||||||
|
},
|
||||||
|
"duration_str": {
|
||||||
|
"type": "keyword",
|
||||||
|
"index": false
|
||||||
|
},
|
||||||
|
"watched": {
|
||||||
|
"type": "boolean"
|
||||||
|
},
|
||||||
|
"watched_date": {
|
||||||
|
"type": "date",
|
||||||
|
"format": "epoch_second"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
@ -239,10 +296,35 @@
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"streams": {
|
||||||
|
"properties": {
|
||||||
|
"type": {
|
||||||
|
"type": "keyword",
|
||||||
|
"index": false
|
||||||
|
},
|
||||||
|
"index": {
|
||||||
|
"type": "short",
|
||||||
|
"index": false
|
||||||
|
},
|
||||||
|
"codec": {
|
||||||
|
"type": "text"
|
||||||
|
},
|
||||||
|
"width": {
|
||||||
|
"type": "short"
|
||||||
|
},
|
||||||
|
"height": {
|
||||||
|
"type": "short"
|
||||||
|
},
|
||||||
|
"bitrate": {
|
||||||
|
"type": "integer"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
"sponsorblock": {
|
"sponsorblock": {
|
||||||
"properties": {
|
"properties": {
|
||||||
"last_refresh": {
|
"last_refresh": {
|
||||||
"type": "date"
|
"type": "date",
|
||||||
|
"format": "epoch_second"
|
||||||
},
|
},
|
||||||
"has_unlocked": {
|
"has_unlocked": {
|
||||||
"type": "boolean"
|
"type": "boolean"
|
||||||
|
@ -250,28 +332,28 @@
|
||||||
"is_enabled": {
|
"is_enabled": {
|
||||||
"type": "boolean"
|
"type": "boolean"
|
||||||
},
|
},
|
||||||
"segments" : {
|
"segments": {
|
||||||
"properties" : {
|
"properties": {
|
||||||
"UUID" : {
|
"UUID": {
|
||||||
"type": "keyword"
|
"type": "keyword"
|
||||||
},
|
},
|
||||||
"actionType" : {
|
"actionType": {
|
||||||
"type": "keyword"
|
"type": "keyword"
|
||||||
},
|
},
|
||||||
"category" : {
|
"category": {
|
||||||
"type": "keyword"
|
"type": "keyword"
|
||||||
},
|
},
|
||||||
"locked" : {
|
"locked": {
|
||||||
"type" : "short"
|
"type": "short"
|
||||||
},
|
},
|
||||||
"segment" : {
|
"segment": {
|
||||||
"type" : "float"
|
"type": "float"
|
||||||
},
|
},
|
||||||
"videoDuration" : {
|
"videoDuration": {
|
||||||
"type" : "float"
|
"type": "float"
|
||||||
},
|
},
|
||||||
"votes" : {
|
"votes": {
|
||||||
"type" : "long"
|
"type": "long"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -294,7 +376,8 @@
|
||||||
"index_name": "download",
|
"index_name": "download",
|
||||||
"expected_map": {
|
"expected_map": {
|
||||||
"timestamp": {
|
"timestamp": {
|
||||||
"type": "date"
|
"type": "date",
|
||||||
|
"format": "epoch_second"
|
||||||
},
|
},
|
||||||
"channel_id": {
|
"channel_id": {
|
||||||
"type": "keyword"
|
"type": "keyword"
|
||||||
|
@ -330,6 +413,12 @@
|
||||||
},
|
},
|
||||||
"vid_type": {
|
"vid_type": {
|
||||||
"type": "keyword"
|
"type": "keyword"
|
||||||
|
},
|
||||||
|
"auto_start": {
|
||||||
|
"type": "boolean"
|
||||||
|
},
|
||||||
|
"message": {
|
||||||
|
"type": "text"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"expected_set": {
|
"expected_set": {
|
||||||
|
@ -386,7 +475,43 @@
|
||||||
"type": "keyword"
|
"type": "keyword"
|
||||||
},
|
},
|
||||||
"playlist_last_refresh": {
|
"playlist_last_refresh": {
|
||||||
"type": "date"
|
"type": "date",
|
||||||
|
"format": "epoch_second"
|
||||||
|
},
|
||||||
|
"playlist_entries": {
|
||||||
|
"properties": {
|
||||||
|
"downloaded": {
|
||||||
|
"type": "boolean"
|
||||||
|
},
|
||||||
|
"idx": {
|
||||||
|
"type": "long"
|
||||||
|
},
|
||||||
|
"title": {
|
||||||
|
"type": "text",
|
||||||
|
"analyzer": "english",
|
||||||
|
"fields": {
|
||||||
|
"keyword": {
|
||||||
|
"type": "keyword",
|
||||||
|
"ignore_above": 256,
|
||||||
|
"normalizer": "to_lower"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"uploader": {
|
||||||
|
"type": "text",
|
||||||
|
"analyzer": "english",
|
||||||
|
"fields": {
|
||||||
|
"keyword": {
|
||||||
|
"type": "keyword",
|
||||||
|
"ignore_above": 256,
|
||||||
|
"normalizer": "to_lower"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"youtube_id": {
|
||||||
|
"type": "keyword"
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"expected_set": {
|
"expected_set": {
|
||||||
|
@ -440,10 +565,11 @@
|
||||||
"type": "text"
|
"type": "text"
|
||||||
},
|
},
|
||||||
"subtitle_last_refresh": {
|
"subtitle_last_refresh": {
|
||||||
"type": "date"
|
"type": "date",
|
||||||
|
"format": "epoch_second"
|
||||||
},
|
},
|
||||||
"subtitle_index": {
|
"subtitle_index": {
|
||||||
"type" : "long"
|
"type": "long"
|
||||||
},
|
},
|
||||||
"subtitle_lang": {
|
"subtitle_lang": {
|
||||||
"type": "keyword"
|
"type": "keyword"
|
||||||
|
@ -452,7 +578,7 @@
|
||||||
"type": "keyword"
|
"type": "keyword"
|
||||||
},
|
},
|
||||||
"subtitle_line": {
|
"subtitle_line": {
|
||||||
"type" : "text",
|
"type": "text",
|
||||||
"analyzer": "english"
|
"analyzer": "english"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
@ -475,7 +601,8 @@
|
||||||
"type": "keyword"
|
"type": "keyword"
|
||||||
},
|
},
|
||||||
"comment_last_refresh": {
|
"comment_last_refresh": {
|
||||||
"type": "date"
|
"type": "date",
|
||||||
|
"format": "epoch_second"
|
||||||
},
|
},
|
||||||
"comment_channel_id": {
|
"comment_channel_id": {
|
||||||
"type": "keyword"
|
"type": "keyword"
|
||||||
|
@ -486,13 +613,14 @@
|
||||||
"type": "keyword"
|
"type": "keyword"
|
||||||
},
|
},
|
||||||
"comment_text": {
|
"comment_text": {
|
||||||
"type" : "text"
|
"type": "text"
|
||||||
},
|
},
|
||||||
"comment_timestamp": {
|
"comment_timestamp": {
|
||||||
"type": "date"
|
"type": "date",
|
||||||
|
"format": "epoch_second"
|
||||||
},
|
},
|
||||||
"comment_time_text": {
|
"comment_time_text": {
|
||||||
"type" : "text"
|
"type": "text"
|
||||||
},
|
},
|
||||||
"comment_likecount": {
|
"comment_likecount": {
|
||||||
"type": "long"
|
"type": "long"
|
||||||
|
|
|
@ -4,12 +4,12 @@ functionality:
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from os import environ
|
|
||||||
from time import sleep
|
from time import sleep
|
||||||
from zoneinfo import ZoneInfo
|
from zoneinfo import ZoneInfo
|
||||||
|
|
||||||
from home.src.es.connect import ElasticWrap
|
from home.src.es.connect import ElasticWrap
|
||||||
from home.src.ta.helper import get_mapping
|
from home.src.ta.helper import get_mapping
|
||||||
|
from home.src.ta.settings import EnvironmentSettings
|
||||||
|
|
||||||
|
|
||||||
class ElasticSnapshot:
|
class ElasticSnapshot:
|
||||||
|
@ -19,7 +19,7 @@ class ElasticSnapshot:
|
||||||
REPO_SETTINGS = {
|
REPO_SETTINGS = {
|
||||||
"compress": "true",
|
"compress": "true",
|
||||||
"chunk_size": "1g",
|
"chunk_size": "1g",
|
||||||
"location": "/usr/share/elasticsearch/data/snapshot",
|
"location": EnvironmentSettings.ES_SNAPSHOT_DIR,
|
||||||
}
|
}
|
||||||
POLICY = "ta_daily"
|
POLICY = "ta_daily"
|
||||||
|
|
||||||
|
@ -254,7 +254,7 @@ class ElasticSnapshot:
|
||||||
expected_format = "%Y-%m-%dT%H:%M:%S.%fZ"
|
expected_format = "%Y-%m-%dT%H:%M:%S.%fZ"
|
||||||
date = datetime.strptime(date_utc, expected_format)
|
date = datetime.strptime(date_utc, expected_format)
|
||||||
local_datetime = date.replace(tzinfo=ZoneInfo("localtime"))
|
local_datetime = date.replace(tzinfo=ZoneInfo("localtime"))
|
||||||
converted = local_datetime.astimezone(ZoneInfo(environ.get("TZ")))
|
converted = local_datetime.astimezone(ZoneInfo(EnvironmentSettings.TZ))
|
||||||
converted_str = converted.strftime("%Y-%m-%d %H:%M")
|
converted_str = converted.strftime("%Y-%m-%d %H:%M")
|
||||||
|
|
||||||
return converted_str
|
return converted_str
|
||||||
|
|
|
@ -1,154 +0,0 @@
|
||||||
"""
|
|
||||||
Functionality:
|
|
||||||
- collection of functions and tasks from frontend
|
|
||||||
- called via user input
|
|
||||||
"""
|
|
||||||
|
|
||||||
from home.src.download.subscriptions import (
|
|
||||||
ChannelSubscription,
|
|
||||||
PlaylistSubscription,
|
|
||||||
)
|
|
||||||
from home.src.index.playlist import YoutubePlaylist
|
|
||||||
from home.src.ta.ta_redis import RedisArchivist
|
|
||||||
from home.src.ta.urlparser import Parser
|
|
||||||
from home.tasks import run_restore_backup, subscribe_to
|
|
||||||
|
|
||||||
|
|
||||||
class PostData:
|
|
||||||
"""
|
|
||||||
map frontend http post values to backend funcs
|
|
||||||
handover long running tasks to celery
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, post_dict, current_user):
|
|
||||||
self.post_dict = post_dict
|
|
||||||
self.to_exec, self.exec_val = list(post_dict.items())[0]
|
|
||||||
self.current_user = current_user
|
|
||||||
|
|
||||||
def run_task(self):
|
|
||||||
"""execute and return task result"""
|
|
||||||
to_exec = self.exec_map()
|
|
||||||
task_result = to_exec()
|
|
||||||
return task_result
|
|
||||||
|
|
||||||
def exec_map(self):
|
|
||||||
"""map dict key and return function to execute"""
|
|
||||||
exec_map = {
|
|
||||||
"change_view": self._change_view,
|
|
||||||
"change_grid": self._change_grid,
|
|
||||||
"unsubscribe": self._unsubscribe,
|
|
||||||
"subscribe": self._subscribe,
|
|
||||||
"sort_order": self._sort_order,
|
|
||||||
"hide_watched": self._hide_watched,
|
|
||||||
"show_subed_only": self._show_subed_only,
|
|
||||||
"show_ignored_only": self._show_ignored_only,
|
|
||||||
"db-restore": self._db_restore,
|
|
||||||
"delete-playlist": self._delete_playlist,
|
|
||||||
}
|
|
||||||
|
|
||||||
return exec_map[self.to_exec]
|
|
||||||
|
|
||||||
def _change_view(self):
|
|
||||||
"""process view changes in home, channel, and downloads"""
|
|
||||||
origin, new_view = self.exec_val.split(":")
|
|
||||||
key = f"{self.current_user}:view:{origin}"
|
|
||||||
print(f"change view: {key} to {new_view}")
|
|
||||||
RedisArchivist().set_message(key, {"status": new_view})
|
|
||||||
return {"success": True}
|
|
||||||
|
|
||||||
def _change_grid(self):
|
|
||||||
"""process change items in grid"""
|
|
||||||
grid_items = int(self.exec_val)
|
|
||||||
grid_items = max(grid_items, 3)
|
|
||||||
grid_items = min(grid_items, 7)
|
|
||||||
|
|
||||||
key = f"{self.current_user}:grid_items"
|
|
||||||
print(f"change grid items: {grid_items}")
|
|
||||||
RedisArchivist().set_message(key, {"status": grid_items})
|
|
||||||
return {"success": True}
|
|
||||||
|
|
||||||
def _unsubscribe(self):
|
|
||||||
"""unsubscribe from channels or playlists"""
|
|
||||||
id_unsub = self.exec_val
|
|
||||||
print(f"{id_unsub}: unsubscribe")
|
|
||||||
to_unsub_list = Parser(id_unsub).parse()
|
|
||||||
for to_unsub in to_unsub_list:
|
|
||||||
unsub_type = to_unsub["type"]
|
|
||||||
unsub_id = to_unsub["url"]
|
|
||||||
if unsub_type == "playlist":
|
|
||||||
PlaylistSubscription().change_subscribe(
|
|
||||||
unsub_id, subscribe_status=False
|
|
||||||
)
|
|
||||||
elif unsub_type == "channel":
|
|
||||||
ChannelSubscription().change_subscribe(
|
|
||||||
unsub_id, channel_subscribed=False
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
raise ValueError("failed to process " + id_unsub)
|
|
||||||
|
|
||||||
return {"success": True}
|
|
||||||
|
|
||||||
def _subscribe(self):
|
|
||||||
"""subscribe to channel or playlist, called from js buttons"""
|
|
||||||
id_sub = self.exec_val
|
|
||||||
print(f"{id_sub}: subscribe")
|
|
||||||
subscribe_to.delay(id_sub)
|
|
||||||
return {"success": True}
|
|
||||||
|
|
||||||
def _sort_order(self):
|
|
||||||
"""change the sort between published to downloaded"""
|
|
||||||
sort_order = {"status": self.exec_val}
|
|
||||||
if self.exec_val in ["asc", "desc"]:
|
|
||||||
RedisArchivist().set_message(
|
|
||||||
f"{self.current_user}:sort_order", sort_order
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
RedisArchivist().set_message(
|
|
||||||
f"{self.current_user}:sort_by", sort_order
|
|
||||||
)
|
|
||||||
return {"success": True}
|
|
||||||
|
|
||||||
def _hide_watched(self):
|
|
||||||
"""toggle if to show watched vids or not"""
|
|
||||||
key = f"{self.current_user}:hide_watched"
|
|
||||||
message = {"status": bool(int(self.exec_val))}
|
|
||||||
print(f"toggle {key}: {message}")
|
|
||||||
RedisArchivist().set_message(key, message)
|
|
||||||
return {"success": True}
|
|
||||||
|
|
||||||
def _show_subed_only(self):
|
|
||||||
"""show or hide subscribed channels only on channels page"""
|
|
||||||
key = f"{self.current_user}:show_subed_only"
|
|
||||||
message = {"status": bool(int(self.exec_val))}
|
|
||||||
print(f"toggle {key}: {message}")
|
|
||||||
RedisArchivist().set_message(key, message)
|
|
||||||
return {"success": True}
|
|
||||||
|
|
||||||
def _show_ignored_only(self):
|
|
||||||
"""switch view on /downloads/ to show ignored only"""
|
|
||||||
show_value = self.exec_val
|
|
||||||
key = f"{self.current_user}:show_ignored_only"
|
|
||||||
value = {"status": show_value}
|
|
||||||
print(f"Filter download view ignored only: {show_value}")
|
|
||||||
RedisArchivist().set_message(key, value)
|
|
||||||
return {"success": True}
|
|
||||||
|
|
||||||
def _db_restore(self):
|
|
||||||
"""restore es zip from settings page"""
|
|
||||||
print("restoring index from backup zip")
|
|
||||||
filename = self.exec_val
|
|
||||||
run_restore_backup.delay(filename)
|
|
||||||
return {"success": True}
|
|
||||||
|
|
||||||
def _delete_playlist(self):
|
|
||||||
"""delete playlist, only metadata or incl all videos"""
|
|
||||||
playlist_dict = self.exec_val
|
|
||||||
playlist_id = playlist_dict["playlist-id"]
|
|
||||||
playlist_action = playlist_dict["playlist-action"]
|
|
||||||
print(f"{playlist_id}: delete playlist {playlist_action}")
|
|
||||||
if playlist_action == "metadata":
|
|
||||||
YoutubePlaylist(playlist_id).delete_metadata()
|
|
||||||
elif playlist_action == "all":
|
|
||||||
YoutubePlaylist(playlist_id).delete_videos_playlist()
|
|
||||||
|
|
||||||
return {"success": True}
|
|
|
@ -2,9 +2,12 @@
|
||||||
- hold all form classes used in the views
|
- hold all form classes used in the views
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
from django import forms
|
from django import forms
|
||||||
from django.contrib.auth.forms import AuthenticationForm
|
from django.contrib.auth.forms import AuthenticationForm
|
||||||
from django.forms.widgets import PasswordInput, TextInput
|
from django.forms.widgets import PasswordInput, TextInput
|
||||||
|
from home.src.ta.helper import get_stylesheets
|
||||||
|
|
||||||
|
|
||||||
class CustomAuthForm(AuthenticationForm):
|
class CustomAuthForm(AuthenticationForm):
|
||||||
|
@ -29,14 +32,16 @@ class CustomAuthForm(AuthenticationForm):
|
||||||
class UserSettingsForm(forms.Form):
|
class UserSettingsForm(forms.Form):
|
||||||
"""user configurations values"""
|
"""user configurations values"""
|
||||||
|
|
||||||
CHOICES = [
|
STYLESHEET_CHOICES = [("", "-- change stylesheet --")]
|
||||||
("", "-- change color scheme --"),
|
STYLESHEET_CHOICES.extend(
|
||||||
("dark", "Dark"),
|
[
|
||||||
("light", "Light"),
|
(stylesheet, os.path.splitext(stylesheet)[0].title())
|
||||||
]
|
for stylesheet in get_stylesheets()
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
colors = forms.ChoiceField(
|
stylesheet = forms.ChoiceField(
|
||||||
widget=forms.Select, choices=CHOICES, required=False
|
widget=forms.Select, choices=STYLESHEET_CHOICES, required=False
|
||||||
)
|
)
|
||||||
page_size = forms.IntegerField(required=False)
|
page_size = forms.IntegerField(required=False)
|
||||||
|
|
||||||
|
@ -44,6 +49,12 @@ class UserSettingsForm(forms.Form):
|
||||||
class ApplicationSettingsForm(forms.Form):
|
class ApplicationSettingsForm(forms.Form):
|
||||||
"""handle all application settings"""
|
"""handle all application settings"""
|
||||||
|
|
||||||
|
AUTOSTART_CHOICES = [
|
||||||
|
("", "-- change subscription autostart --"),
|
||||||
|
("0", "disable auto start"),
|
||||||
|
("1", "enable auto start"),
|
||||||
|
]
|
||||||
|
|
||||||
METADATA_CHOICES = [
|
METADATA_CHOICES = [
|
||||||
("", "-- change metadata embed --"),
|
("", "-- change metadata embed --"),
|
||||||
("0", "don't embed metadata"),
|
("0", "don't embed metadata"),
|
||||||
|
@ -94,8 +105,8 @@ class ApplicationSettingsForm(forms.Form):
|
||||||
|
|
||||||
COOKIE_IMPORT_CHOICES = [
|
COOKIE_IMPORT_CHOICES = [
|
||||||
("", "-- change cookie settings"),
|
("", "-- change cookie settings"),
|
||||||
("0", "disable cookie"),
|
("0", "remove cookie"),
|
||||||
("1", "enable cookie"),
|
("1", "import cookie"),
|
||||||
]
|
]
|
||||||
|
|
||||||
subscriptions_channel_size = forms.IntegerField(
|
subscriptions_channel_size = forms.IntegerField(
|
||||||
|
@ -107,12 +118,16 @@ class ApplicationSettingsForm(forms.Form):
|
||||||
subscriptions_shorts_channel_size = forms.IntegerField(
|
subscriptions_shorts_channel_size = forms.IntegerField(
|
||||||
required=False, min_value=0
|
required=False, min_value=0
|
||||||
)
|
)
|
||||||
downloads_limit_count = forms.IntegerField(required=False)
|
subscriptions_auto_start = forms.ChoiceField(
|
||||||
|
widget=forms.Select, choices=AUTOSTART_CHOICES, required=False
|
||||||
|
)
|
||||||
downloads_limit_speed = forms.IntegerField(required=False)
|
downloads_limit_speed = forms.IntegerField(required=False)
|
||||||
downloads_throttledratelimit = forms.IntegerField(required=False)
|
downloads_throttledratelimit = forms.IntegerField(required=False)
|
||||||
downloads_sleep_interval = forms.IntegerField(required=False)
|
downloads_sleep_interval = forms.IntegerField(required=False)
|
||||||
downloads_autodelete_days = forms.IntegerField(required=False)
|
downloads_autodelete_days = forms.IntegerField(required=False)
|
||||||
downloads_format = forms.CharField(required=False)
|
downloads_format = forms.CharField(required=False)
|
||||||
|
downloads_format_sort = forms.CharField(required=False)
|
||||||
|
downloads_extractor_lang = forms.CharField(required=False)
|
||||||
downloads_add_metadata = forms.ChoiceField(
|
downloads_add_metadata = forms.ChoiceField(
|
||||||
widget=forms.Select, choices=METADATA_CHOICES, required=False
|
widget=forms.Select, choices=METADATA_CHOICES, required=False
|
||||||
)
|
)
|
||||||
|
@ -144,18 +159,6 @@ class ApplicationSettingsForm(forms.Form):
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class SchedulerSettingsForm(forms.Form):
|
|
||||||
"""handle scheduler settings"""
|
|
||||||
|
|
||||||
update_subscribed = forms.CharField(required=False)
|
|
||||||
download_pending = forms.CharField(required=False)
|
|
||||||
check_reindex = forms.CharField(required=False)
|
|
||||||
check_reindex_days = forms.IntegerField(required=False)
|
|
||||||
thumbnail_check = forms.CharField(required=False)
|
|
||||||
run_backup = forms.CharField(required=False)
|
|
||||||
run_backup_rotate = forms.IntegerField(required=False)
|
|
||||||
|
|
||||||
|
|
||||||
class MultiSearchForm(forms.Form):
|
class MultiSearchForm(forms.Form):
|
||||||
"""multi search form for /search/"""
|
"""multi search form for /search/"""
|
||||||
|
|
||||||
|
@ -218,6 +221,20 @@ class SubscribeToPlaylistForm(forms.Form):
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class CreatePlaylistForm(forms.Form):
|
||||||
|
"""text area form to create a single custom playlist"""
|
||||||
|
|
||||||
|
create = forms.CharField(
|
||||||
|
label="Or create custom playlist",
|
||||||
|
widget=forms.Textarea(
|
||||||
|
attrs={
|
||||||
|
"rows": 1,
|
||||||
|
"placeholder": "Input playlist name",
|
||||||
|
}
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class ChannelOverwriteForm(forms.Form):
|
class ChannelOverwriteForm(forms.Form):
|
||||||
"""custom overwrites for channel settings"""
|
"""custom overwrites for channel settings"""
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,101 @@
|
||||||
|
"""
|
||||||
|
Functionality:
|
||||||
|
- handle schedule forms
|
||||||
|
- implement form validation
|
||||||
|
"""
|
||||||
|
|
||||||
|
from celery.schedules import crontab
|
||||||
|
from django import forms
|
||||||
|
from home.src.ta.task_config import TASK_CONFIG
|
||||||
|
|
||||||
|
|
||||||
|
class CrontabValidator:
|
||||||
|
"""validate crontab"""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def validate_fields(cron_fields):
|
||||||
|
"""expect 3 cron fields"""
|
||||||
|
if not len(cron_fields) == 3:
|
||||||
|
raise forms.ValidationError("expected three cron schedule fields")
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def validate_minute(minute_field):
|
||||||
|
"""expect minute int"""
|
||||||
|
try:
|
||||||
|
minute_value = int(minute_field)
|
||||||
|
if not 0 <= minute_value <= 59:
|
||||||
|
raise forms.ValidationError(
|
||||||
|
"Invalid value for minutes. Must be between 0 and 59."
|
||||||
|
)
|
||||||
|
except ValueError as err:
|
||||||
|
raise forms.ValidationError(
|
||||||
|
"Invalid value for minutes. Must be an integer."
|
||||||
|
) from err
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def validate_cron_tab(minute, hour, day_of_week):
|
||||||
|
"""check if crontab can be created"""
|
||||||
|
try:
|
||||||
|
crontab(minute=minute, hour=hour, day_of_week=day_of_week)
|
||||||
|
except ValueError as err:
|
||||||
|
raise forms.ValidationError(f"invalid crontab: {err}") from err
|
||||||
|
|
||||||
|
def validate(self, cron_expression):
|
||||||
|
"""create crontab schedule"""
|
||||||
|
if cron_expression == "auto":
|
||||||
|
return
|
||||||
|
|
||||||
|
cron_fields = cron_expression.split()
|
||||||
|
self.validate_fields(cron_fields)
|
||||||
|
|
||||||
|
minute, hour, day_of_week = cron_fields
|
||||||
|
self.validate_minute(minute)
|
||||||
|
self.validate_cron_tab(minute, hour, day_of_week)
|
||||||
|
|
||||||
|
|
||||||
|
def validate_cron(cron_expression):
|
||||||
|
"""callable for field"""
|
||||||
|
CrontabValidator().validate(cron_expression)
|
||||||
|
|
||||||
|
|
||||||
|
class SchedulerSettingsForm(forms.Form):
|
||||||
|
"""handle scheduler settings"""
|
||||||
|
|
||||||
|
update_subscribed = forms.CharField(
|
||||||
|
required=False, validators=[validate_cron]
|
||||||
|
)
|
||||||
|
download_pending = forms.CharField(
|
||||||
|
required=False, validators=[validate_cron]
|
||||||
|
)
|
||||||
|
check_reindex = forms.CharField(required=False, validators=[validate_cron])
|
||||||
|
check_reindex_days = forms.IntegerField(required=False)
|
||||||
|
thumbnail_check = forms.CharField(
|
||||||
|
required=False, validators=[validate_cron]
|
||||||
|
)
|
||||||
|
run_backup = forms.CharField(required=False, validators=[validate_cron])
|
||||||
|
run_backup_rotate = forms.IntegerField(required=False)
|
||||||
|
|
||||||
|
|
||||||
|
class NotificationSettingsForm(forms.Form):
|
||||||
|
"""add notification URL"""
|
||||||
|
|
||||||
|
SUPPORTED_TASKS = [
|
||||||
|
"update_subscribed",
|
||||||
|
"extract_download",
|
||||||
|
"download_pending",
|
||||||
|
"check_reindex",
|
||||||
|
]
|
||||||
|
TASK_LIST = [(i, TASK_CONFIG[i]["title"]) for i in SUPPORTED_TASKS]
|
||||||
|
|
||||||
|
TASK_CHOICES = [("", "-- select task --")]
|
||||||
|
TASK_CHOICES.extend(TASK_LIST)
|
||||||
|
|
||||||
|
PLACEHOLDER = "Apprise notification URL"
|
||||||
|
|
||||||
|
task = forms.ChoiceField(
|
||||||
|
widget=forms.Select, choices=TASK_CHOICES, required=False
|
||||||
|
)
|
||||||
|
notification_url = forms.CharField(
|
||||||
|
required=False,
|
||||||
|
widget=forms.TextInput(attrs={"placeholder": PLACEHOLDER}),
|
||||||
|
)
|
|
@ -6,154 +6,18 @@ Functionality:
|
||||||
- calculate pagination values
|
- calculate pagination values
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import urllib.parse
|
from api.src.search_processor import SearchProcess
|
||||||
from datetime import datetime
|
|
||||||
|
|
||||||
from home.src.download.thumbnails import ThumbManager
|
|
||||||
from home.src.es.connect import ElasticWrap
|
from home.src.es.connect import ElasticWrap
|
||||||
from home.src.ta.config import AppConfig
|
|
||||||
|
|
||||||
|
|
||||||
class SearchHandler:
|
|
||||||
"""search elastic search"""
|
|
||||||
|
|
||||||
def __init__(self, path, config, data=False):
|
|
||||||
self.max_hits = None
|
|
||||||
self.path = path
|
|
||||||
self.config = config
|
|
||||||
self.data = data
|
|
||||||
|
|
||||||
def get_data(self):
|
|
||||||
"""get the data"""
|
|
||||||
response, _ = ElasticWrap(self.path, config=self.config).get(self.data)
|
|
||||||
|
|
||||||
if "hits" in response.keys():
|
|
||||||
self.max_hits = response["hits"]["total"]["value"]
|
|
||||||
return_value = response["hits"]["hits"]
|
|
||||||
else:
|
|
||||||
# simulate list for single result to reuse rest of class
|
|
||||||
return_value = [response]
|
|
||||||
|
|
||||||
# stop if empty
|
|
||||||
if not return_value:
|
|
||||||
return False
|
|
||||||
|
|
||||||
all_videos = []
|
|
||||||
all_channels = []
|
|
||||||
for idx, hit in enumerate(return_value):
|
|
||||||
return_value[idx] = self.hit_cleanup(hit)
|
|
||||||
if hit["_index"] == "ta_video":
|
|
||||||
video_dict, channel_dict = self.vid_cache_link(hit)
|
|
||||||
if video_dict not in all_videos:
|
|
||||||
all_videos.append(video_dict)
|
|
||||||
if channel_dict not in all_channels:
|
|
||||||
all_channels.append(channel_dict)
|
|
||||||
elif hit["_index"] == "ta_channel":
|
|
||||||
channel_dict = self.channel_cache_link(hit)
|
|
||||||
if channel_dict not in all_channels:
|
|
||||||
all_channels.append(channel_dict)
|
|
||||||
|
|
||||||
return return_value
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def vid_cache_link(hit):
|
|
||||||
"""download thumbnails into cache"""
|
|
||||||
vid_thumb = hit["source"]["vid_thumb_url"]
|
|
||||||
youtube_id = hit["source"]["youtube_id"]
|
|
||||||
channel_id_hit = hit["source"]["channel"]["channel_id"]
|
|
||||||
chan_thumb = hit["source"]["channel"]["channel_thumb_url"]
|
|
||||||
try:
|
|
||||||
chan_banner = hit["source"]["channel"]["channel_banner_url"]
|
|
||||||
except KeyError:
|
|
||||||
chan_banner = False
|
|
||||||
video_dict = {"youtube_id": youtube_id, "vid_thumb": vid_thumb}
|
|
||||||
channel_dict = {
|
|
||||||
"channel_id": channel_id_hit,
|
|
||||||
"chan_thumb": chan_thumb,
|
|
||||||
"chan_banner": chan_banner,
|
|
||||||
}
|
|
||||||
return video_dict, channel_dict
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def channel_cache_link(hit):
|
|
||||||
"""build channel thumb links"""
|
|
||||||
channel_id_hit = hit["source"]["channel_id"]
|
|
||||||
chan_thumb = hit["source"]["channel_thumb_url"]
|
|
||||||
try:
|
|
||||||
chan_banner = hit["source"]["channel_banner_url"]
|
|
||||||
except KeyError:
|
|
||||||
chan_banner = False
|
|
||||||
channel_dict = {
|
|
||||||
"channel_id": channel_id_hit,
|
|
||||||
"chan_thumb": chan_thumb,
|
|
||||||
"chan_banner": chan_banner,
|
|
||||||
}
|
|
||||||
return channel_dict
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def hit_cleanup(hit):
|
|
||||||
"""clean up and parse data from a single hit"""
|
|
||||||
hit["source"] = hit.pop("_source")
|
|
||||||
hit_keys = hit["source"].keys()
|
|
||||||
if "media_url" in hit_keys:
|
|
||||||
parsed_url = urllib.parse.quote(hit["source"]["media_url"])
|
|
||||||
hit["source"]["media_url"] = parsed_url
|
|
||||||
|
|
||||||
if "published" in hit_keys:
|
|
||||||
published = hit["source"]["published"]
|
|
||||||
date_pub = datetime.strptime(published, "%Y-%m-%d")
|
|
||||||
date_str = datetime.strftime(date_pub, "%d %b, %Y")
|
|
||||||
hit["source"]["published"] = date_str
|
|
||||||
|
|
||||||
if "vid_last_refresh" in hit_keys:
|
|
||||||
vid_last_refresh = hit["source"]["vid_last_refresh"]
|
|
||||||
date_refresh = datetime.fromtimestamp(vid_last_refresh)
|
|
||||||
date_str = datetime.strftime(date_refresh, "%d %b, %Y")
|
|
||||||
hit["source"]["vid_last_refresh"] = date_str
|
|
||||||
|
|
||||||
if "playlist_last_refresh" in hit_keys:
|
|
||||||
playlist_last_refresh = hit["source"]["playlist_last_refresh"]
|
|
||||||
date_refresh = datetime.fromtimestamp(playlist_last_refresh)
|
|
||||||
date_str = datetime.strftime(date_refresh, "%d %b, %Y")
|
|
||||||
hit["source"]["playlist_last_refresh"] = date_str
|
|
||||||
|
|
||||||
if "vid_thumb_url" in hit_keys:
|
|
||||||
youtube_id = hit["source"]["youtube_id"]
|
|
||||||
thumb_path = ThumbManager(youtube_id).vid_thumb_path()
|
|
||||||
hit["source"]["vid_thumb_url"] = f"/cache/{thumb_path}"
|
|
||||||
|
|
||||||
if "channel_last_refresh" in hit_keys:
|
|
||||||
refreshed = hit["source"]["channel_last_refresh"]
|
|
||||||
date_refresh = datetime.fromtimestamp(refreshed)
|
|
||||||
date_str = datetime.strftime(date_refresh, "%d %b, %Y")
|
|
||||||
hit["source"]["channel_last_refresh"] = date_str
|
|
||||||
|
|
||||||
if "channel" in hit_keys:
|
|
||||||
channel_keys = hit["source"]["channel"].keys()
|
|
||||||
if "channel_last_refresh" in channel_keys:
|
|
||||||
refreshed = hit["source"]["channel"]["channel_last_refresh"]
|
|
||||||
date_refresh = datetime.fromtimestamp(refreshed)
|
|
||||||
date_str = datetime.strftime(date_refresh, "%d %b, %Y")
|
|
||||||
hit["source"]["channel"]["channel_last_refresh"] = date_str
|
|
||||||
|
|
||||||
if "subtitle_fragment_id" in hit_keys:
|
|
||||||
youtube_id = hit["source"]["youtube_id"]
|
|
||||||
thumb_path = ThumbManager(youtube_id).vid_thumb_path()
|
|
||||||
hit["source"]["vid_thumb_url"] = f"/cache/{thumb_path}"
|
|
||||||
|
|
||||||
return hit
|
|
||||||
|
|
||||||
|
|
||||||
class SearchForm:
|
class SearchForm:
|
||||||
"""build query from search form data"""
|
"""build query from search form data"""
|
||||||
|
|
||||||
CONFIG = AppConfig().config
|
|
||||||
|
|
||||||
def multi_search(self, search_query):
|
def multi_search(self, search_query):
|
||||||
"""searching through index"""
|
"""searching through index"""
|
||||||
path, query, query_type = SearchParser(search_query).run()
|
path, query, query_type = SearchParser(search_query).run()
|
||||||
look_up = SearchHandler(path, config=self.CONFIG, data=query)
|
response, _ = ElasticWrap(path).get(data=query)
|
||||||
search_results = look_up.get_data()
|
search_results = SearchProcess(response).process()
|
||||||
all_results = self.build_results(search_results)
|
all_results = self.build_results(search_results)
|
||||||
|
|
||||||
return {"results": all_results, "queryType": query_type}
|
return {"results": all_results, "queryType": query_type}
|
||||||
|
@ -429,6 +293,7 @@ class QueryBuilder:
|
||||||
"channel_name._2gram^2",
|
"channel_name._2gram^2",
|
||||||
"channel_name._3gram^2",
|
"channel_name._3gram^2",
|
||||||
"channel_name.search_as_you_type^2",
|
"channel_name.search_as_you_type^2",
|
||||||
|
"channel_tags",
|
||||||
],
|
],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -502,7 +367,6 @@ class QueryBuilder:
|
||||||
|
|
||||||
query = {
|
query = {
|
||||||
"size": 30,
|
"size": 30,
|
||||||
"_source": {"excludes": "subtitle_line"},
|
|
||||||
"query": {"bool": {"must": must_list}},
|
"query": {"bool": {"must": must_list}},
|
||||||
"highlight": {
|
"highlight": {
|
||||||
"fields": {
|
"fields": {
|
||||||
|
|
|
@ -10,154 +10,40 @@ import re
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
from bs4 import BeautifulSoup
|
|
||||||
from home.src.download import queue # partial import
|
|
||||||
from home.src.download.thumbnails import ThumbManager
|
from home.src.download.thumbnails import ThumbManager
|
||||||
from home.src.download.yt_dlp_base import YtWrap
|
from home.src.download.yt_dlp_base import YtWrap
|
||||||
from home.src.es.connect import ElasticWrap, IndexPaginate
|
from home.src.es.connect import ElasticWrap, IndexPaginate
|
||||||
from home.src.index.generic import YouTubeItem
|
from home.src.index.generic import YouTubeItem
|
||||||
from home.src.index.playlist import YoutubePlaylist
|
from home.src.index.playlist import YoutubePlaylist
|
||||||
from home.src.ta.helper import clean_string, requests_headers
|
from home.src.ta.helper import requests_headers
|
||||||
|
from home.src.ta.settings import EnvironmentSettings
|
||||||
|
|
||||||
|
|
||||||
class ChannelScraper:
|
def banner_extractor(channel_id: str) -> dict[str, str] | None:
|
||||||
"""custom scraper using bs4 to scrape channel about page
|
"""workaround for new channel renderer, upstream #9893"""
|
||||||
will be able to be integrated into yt-dlp
|
url = f"https://www.youtube.com/channel/{channel_id}?hl=en"
|
||||||
once #2237 and #2350 are merged upstream
|
cookies = {"SOCS": "CAI"}
|
||||||
"""
|
response = requests.get(
|
||||||
|
url, cookies=cookies, headers=requests_headers(), timeout=30
|
||||||
|
)
|
||||||
|
if not response.ok:
|
||||||
|
return None
|
||||||
|
|
||||||
def __init__(self, channel_id):
|
matched_urls = re.findall(
|
||||||
self.channel_id = channel_id
|
r'"(https://yt3.googleusercontent.com/[^"]+=w(\d{3,4})-fcrop64[^"]*)"',
|
||||||
self.soup = False
|
response.text,
|
||||||
self.yt_json = False
|
)
|
||||||
self.json_data = False
|
if not matched_urls:
|
||||||
|
return None
|
||||||
|
|
||||||
def get_json(self):
|
sorted_urls = sorted(matched_urls, key=lambda x: int(x[1]), reverse=True)
|
||||||
"""main method to return channel dict"""
|
banner = sorted_urls[0][0]
|
||||||
self.get_soup()
|
channel_art_fallback = {
|
||||||
self._extract_yt_json()
|
"channel_banner_url": banner,
|
||||||
if self._is_deactivated():
|
"channel_tvart_url": banner.split("-fcrop64")[0],
|
||||||
return False
|
}
|
||||||
|
|
||||||
self._parse_channel_main()
|
return channel_art_fallback
|
||||||
self._parse_channel_meta()
|
|
||||||
return self.json_data
|
|
||||||
|
|
||||||
def get_soup(self):
|
|
||||||
"""return soup from youtube"""
|
|
||||||
print(f"{self.channel_id}: scrape channel data from youtube")
|
|
||||||
url = f"https://www.youtube.com/channel/{self.channel_id}/about?hl=en"
|
|
||||||
cookies = {"CONSENT": "YES+xxxxxxxxxxxxxxxxxxxxxxxxxxx"}
|
|
||||||
response = requests.get(
|
|
||||||
url, cookies=cookies, headers=requests_headers(), timeout=10
|
|
||||||
)
|
|
||||||
if response.ok:
|
|
||||||
channel_page = response.text
|
|
||||||
else:
|
|
||||||
print(f"{self.channel_id}: failed to extract channel info")
|
|
||||||
raise ConnectionError
|
|
||||||
self.soup = BeautifulSoup(channel_page, "html.parser")
|
|
||||||
|
|
||||||
def _extract_yt_json(self):
|
|
||||||
"""parse soup and get ytInitialData json"""
|
|
||||||
all_scripts = self.soup.find("body").find_all("script")
|
|
||||||
for script in all_scripts:
|
|
||||||
if "var ytInitialData = " in str(script):
|
|
||||||
script_content = str(script)
|
|
||||||
break
|
|
||||||
# extract payload
|
|
||||||
script_content = script_content.split("var ytInitialData = ")[1]
|
|
||||||
json_raw = script_content.rstrip(";</script>")
|
|
||||||
self.yt_json = json.loads(json_raw)
|
|
||||||
|
|
||||||
def _is_deactivated(self):
|
|
||||||
"""check if channel is deactivated"""
|
|
||||||
alerts = self.yt_json.get("alerts")
|
|
||||||
if not alerts:
|
|
||||||
return False
|
|
||||||
|
|
||||||
for alert in alerts:
|
|
||||||
alert_text = alert["alertRenderer"]["text"]["simpleText"]
|
|
||||||
print(f"{self.channel_id}: failed to extract, {alert_text}")
|
|
||||||
return True
|
|
||||||
|
|
||||||
def _parse_channel_main(self):
|
|
||||||
"""extract maintab values from scraped channel json data"""
|
|
||||||
main_tab = self.yt_json["header"]["c4TabbedHeaderRenderer"]
|
|
||||||
# build and return dict
|
|
||||||
self.json_data = {
|
|
||||||
"channel_active": True,
|
|
||||||
"channel_last_refresh": int(datetime.now().timestamp()),
|
|
||||||
"channel_subs": self._get_channel_subs(main_tab),
|
|
||||||
"channel_name": main_tab["title"],
|
|
||||||
"channel_banner_url": self._get_thumbnails(main_tab, "banner"),
|
|
||||||
"channel_tvart_url": self._get_thumbnails(main_tab, "tvBanner"),
|
|
||||||
"channel_id": self.channel_id,
|
|
||||||
"channel_subscribed": False,
|
|
||||||
}
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _get_thumbnails(main_tab, thumb_name):
|
|
||||||
"""extract banner url from main_tab"""
|
|
||||||
try:
|
|
||||||
all_banners = main_tab[thumb_name]["thumbnails"]
|
|
||||||
banner = sorted(all_banners, key=lambda k: k["width"])[-1]["url"]
|
|
||||||
except KeyError:
|
|
||||||
banner = False
|
|
||||||
|
|
||||||
return banner
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _get_channel_subs(main_tab):
|
|
||||||
"""process main_tab to get channel subs as int"""
|
|
||||||
try:
|
|
||||||
sub_text_simple = main_tab["subscriberCountText"]["simpleText"]
|
|
||||||
sub_text = sub_text_simple.split(" ")[0]
|
|
||||||
if sub_text[-1] == "K":
|
|
||||||
channel_subs = int(float(sub_text.replace("K", "")) * 1000)
|
|
||||||
elif sub_text[-1] == "M":
|
|
||||||
channel_subs = int(float(sub_text.replace("M", "")) * 1000000)
|
|
||||||
elif int(sub_text) >= 0:
|
|
||||||
channel_subs = int(sub_text)
|
|
||||||
else:
|
|
||||||
message = f"{sub_text} not dealt with"
|
|
||||||
print(message)
|
|
||||||
except KeyError:
|
|
||||||
channel_subs = 0
|
|
||||||
|
|
||||||
return channel_subs
|
|
||||||
|
|
||||||
def _parse_channel_meta(self):
|
|
||||||
"""extract meta tab values from channel payload"""
|
|
||||||
# meta tab
|
|
||||||
meta_tab = self.yt_json["metadata"]["channelMetadataRenderer"]
|
|
||||||
all_thumbs = meta_tab["avatar"]["thumbnails"]
|
|
||||||
thumb_url = sorted(all_thumbs, key=lambda k: k["width"])[-1]["url"]
|
|
||||||
# stats tab
|
|
||||||
renderer = "twoColumnBrowseResultsRenderer"
|
|
||||||
all_tabs = self.yt_json["contents"][renderer]["tabs"]
|
|
||||||
for tab in all_tabs:
|
|
||||||
if "tabRenderer" in tab.keys():
|
|
||||||
if tab["tabRenderer"]["title"] == "About":
|
|
||||||
about_tab = tab["tabRenderer"]["content"][
|
|
||||||
"sectionListRenderer"
|
|
||||||
]["contents"][0]["itemSectionRenderer"]["contents"][0][
|
|
||||||
"channelAboutFullMetadataRenderer"
|
|
||||||
]
|
|
||||||
break
|
|
||||||
try:
|
|
||||||
channel_views_text = about_tab["viewCountText"]["simpleText"]
|
|
||||||
channel_views = int(re.sub(r"\D", "", channel_views_text))
|
|
||||||
except KeyError:
|
|
||||||
channel_views = 0
|
|
||||||
|
|
||||||
self.json_data.update(
|
|
||||||
{
|
|
||||||
"channel_description": meta_tab["description"],
|
|
||||||
"channel_thumb_url": thumb_url,
|
|
||||||
"channel_views": channel_views,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class YoutubeChannel(YouTubeItem):
|
class YoutubeChannel(YouTubeItem):
|
||||||
|
@ -166,10 +52,13 @@ class YoutubeChannel(YouTubeItem):
|
||||||
es_path = False
|
es_path = False
|
||||||
index_name = "ta_channel"
|
index_name = "ta_channel"
|
||||||
yt_base = "https://www.youtube.com/channel/"
|
yt_base = "https://www.youtube.com/channel/"
|
||||||
|
yt_obs = {
|
||||||
|
"playlist_items": "1,0",
|
||||||
|
"skip_download": True,
|
||||||
|
}
|
||||||
|
|
||||||
def __init__(self, youtube_id, task=False):
|
def __init__(self, youtube_id, task=False):
|
||||||
super().__init__(youtube_id)
|
super().__init__(youtube_id)
|
||||||
self.es_path = f"{self.index_name}/_doc/{youtube_id}"
|
|
||||||
self.all_playlists = False
|
self.all_playlists = False
|
||||||
self.task = task
|
self.task = task
|
||||||
|
|
||||||
|
@ -179,23 +68,102 @@ class YoutubeChannel(YouTubeItem):
|
||||||
if self.json_data:
|
if self.json_data:
|
||||||
return
|
return
|
||||||
|
|
||||||
self.get_from_youtube(fallback)
|
self.get_from_youtube()
|
||||||
|
if not self.youtube_meta and fallback:
|
||||||
|
self._video_fallback(fallback)
|
||||||
|
else:
|
||||||
|
self.process_youtube_meta()
|
||||||
|
self.get_channel_art()
|
||||||
|
|
||||||
if upload:
|
if upload:
|
||||||
self.upload_to_es()
|
self.upload_to_es()
|
||||||
return
|
|
||||||
|
|
||||||
def get_from_youtube(self, fallback=False):
|
def process_youtube_meta(self):
|
||||||
"""use bs4 to scrape channel about page"""
|
"""extract relevant fields"""
|
||||||
self.json_data = ChannelScraper(self.youtube_id).get_json()
|
self.youtube_meta["thumbnails"].reverse()
|
||||||
|
self.json_data = {
|
||||||
|
"channel_active": True,
|
||||||
|
"channel_description": self.youtube_meta.get("description", False),
|
||||||
|
"channel_id": self.youtube_id,
|
||||||
|
"channel_last_refresh": int(datetime.now().timestamp()),
|
||||||
|
"channel_name": self.youtube_meta["uploader"],
|
||||||
|
"channel_subs": self._extract_follower_count(),
|
||||||
|
"channel_subscribed": False,
|
||||||
|
"channel_tags": self._parse_tags(self.youtube_meta.get("tags")),
|
||||||
|
"channel_banner_url": self._get_banner_art(),
|
||||||
|
"channel_thumb_url": self._get_thumb_art(),
|
||||||
|
"channel_tvart_url": self._get_tv_art(),
|
||||||
|
"channel_views": self.youtube_meta.get("view_count") or 0,
|
||||||
|
}
|
||||||
|
self._inject_fallback()
|
||||||
|
|
||||||
if not self.json_data and fallback:
|
def _inject_fallback(self):
|
||||||
self._video_fallback(fallback)
|
"""fallback channel art work, workaround for upstream #9893"""
|
||||||
|
if self.json_data["channel_banner_url"]:
|
||||||
if not self.json_data:
|
|
||||||
return
|
return
|
||||||
|
|
||||||
self.get_channel_art()
|
print(f"{self.youtube_id}: attempt art fallback extraction")
|
||||||
|
fallback = banner_extractor(self.youtube_id)
|
||||||
|
if fallback:
|
||||||
|
print(f"{self.youtube_id}: fallback succeeded: {fallback}")
|
||||||
|
self.json_data.update(fallback)
|
||||||
|
|
||||||
|
def _extract_follower_count(self) -> int:
|
||||||
|
"""workaround for upstream #9893, extract subs from first video"""
|
||||||
|
subs = self.youtube_meta.get("channel_follower_count")
|
||||||
|
if subs is not None:
|
||||||
|
return subs
|
||||||
|
|
||||||
|
entries = self.youtube_meta.get("entries", [])
|
||||||
|
if entries:
|
||||||
|
first_entry = entries[0]
|
||||||
|
if isinstance(first_entry, dict):
|
||||||
|
subs_entry = first_entry.get("channel_follower_count")
|
||||||
|
if subs_entry is not None:
|
||||||
|
return subs_entry
|
||||||
|
|
||||||
|
return 0
|
||||||
|
|
||||||
|
def _parse_tags(self, tags):
|
||||||
|
"""parse channel tags"""
|
||||||
|
if not tags:
|
||||||
|
return False
|
||||||
|
|
||||||
|
joined = " ".join(tags)
|
||||||
|
return [i.strip() for i in joined.split('"') if i and not i == " "]
|
||||||
|
|
||||||
|
def _get_thumb_art(self):
|
||||||
|
"""extract thumb art"""
|
||||||
|
for i in self.youtube_meta["thumbnails"]:
|
||||||
|
if not i.get("width"):
|
||||||
|
continue
|
||||||
|
if i.get("width") == i.get("height"):
|
||||||
|
return i["url"]
|
||||||
|
|
||||||
|
return False
|
||||||
|
|
||||||
|
def _get_tv_art(self):
|
||||||
|
"""extract tv artwork"""
|
||||||
|
for i in self.youtube_meta["thumbnails"]:
|
||||||
|
if i.get("id") == "banner_uncropped":
|
||||||
|
return i["url"]
|
||||||
|
for i in self.youtube_meta["thumbnails"]:
|
||||||
|
if not i.get("width"):
|
||||||
|
continue
|
||||||
|
if i["width"] // i["height"] < 2 and not i["width"] == i["height"]:
|
||||||
|
return i["url"]
|
||||||
|
|
||||||
|
return False
|
||||||
|
|
||||||
|
def _get_banner_art(self):
|
||||||
|
"""extract banner artwork"""
|
||||||
|
for i in self.youtube_meta["thumbnails"]:
|
||||||
|
if not i.get("width"):
|
||||||
|
continue
|
||||||
|
if i["width"] // i["height"] > 5:
|
||||||
|
return i["url"]
|
||||||
|
|
||||||
|
return False
|
||||||
|
|
||||||
def _video_fallback(self, fallback):
|
def _video_fallback(self, fallback):
|
||||||
"""use video metadata as fallback"""
|
"""use video metadata as fallback"""
|
||||||
|
@ -209,6 +177,7 @@ class YoutubeChannel(YouTubeItem):
|
||||||
"channel_tvart_url": False,
|
"channel_tvart_url": False,
|
||||||
"channel_id": self.youtube_id,
|
"channel_id": self.youtube_id,
|
||||||
"channel_subscribed": False,
|
"channel_subscribed": False,
|
||||||
|
"channel_tags": False,
|
||||||
"channel_description": False,
|
"channel_description": False,
|
||||||
"channel_thumb_url": False,
|
"channel_thumb_url": False,
|
||||||
"channel_views": 0,
|
"channel_views": 0,
|
||||||
|
@ -218,7 +187,7 @@ class YoutubeChannel(YouTubeItem):
|
||||||
def _info_json_fallback(self):
|
def _info_json_fallback(self):
|
||||||
"""read channel info.json for additional metadata"""
|
"""read channel info.json for additional metadata"""
|
||||||
info_json = os.path.join(
|
info_json = os.path.join(
|
||||||
self.config["application"]["cache_dir"],
|
EnvironmentSettings.CACHE_DIR,
|
||||||
"import",
|
"import",
|
||||||
f"{self.youtube_id}.info.json",
|
f"{self.youtube_id}.info.json",
|
||||||
)
|
)
|
||||||
|
@ -261,12 +230,10 @@ class YoutubeChannel(YouTubeItem):
|
||||||
|
|
||||||
def get_folder_path(self):
|
def get_folder_path(self):
|
||||||
"""get folder where media files get stored"""
|
"""get folder where media files get stored"""
|
||||||
channel_name = self.json_data["channel_name"]
|
folder_path = os.path.join(
|
||||||
folder_name = clean_string(channel_name)
|
EnvironmentSettings.MEDIA_DIR,
|
||||||
if len(folder_name) <= 3:
|
self.json_data["channel_id"],
|
||||||
# fall back to channel id
|
)
|
||||||
folder_name = self.json_data["channel_id"]
|
|
||||||
folder_path = os.path.join(self.app_conf["videos"], folder_name)
|
|
||||||
return folder_path
|
return folder_path
|
||||||
|
|
||||||
def delete_es_videos(self):
|
def delete_es_videos(self):
|
||||||
|
@ -287,12 +254,20 @@ class YoutubeChannel(YouTubeItem):
|
||||||
}
|
}
|
||||||
_, _ = ElasticWrap("ta_comment/_delete_by_query").post(data)
|
_, _ = ElasticWrap("ta_comment/_delete_by_query").post(data)
|
||||||
|
|
||||||
|
def delete_es_subtitles(self):
|
||||||
|
"""delete all subtitles from this channel"""
|
||||||
|
data = {
|
||||||
|
"query": {
|
||||||
|
"term": {"subtitle_channel_id": {"value": self.youtube_id}}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_, _ = ElasticWrap("ta_subtitle/_delete_by_query").post(data)
|
||||||
|
|
||||||
def delete_playlists(self):
|
def delete_playlists(self):
|
||||||
"""delete all indexed playlist from es"""
|
"""delete all indexed playlist from es"""
|
||||||
all_playlists = self.get_indexed_playlists()
|
all_playlists = self.get_indexed_playlists()
|
||||||
for playlist in all_playlists:
|
for playlist in all_playlists:
|
||||||
playlist_id = playlist["playlist_id"]
|
YoutubePlaylist(playlist["playlist_id"]).delete_metadata()
|
||||||
YoutubePlaylist(playlist_id).delete_metadata()
|
|
||||||
|
|
||||||
def delete_channel(self):
|
def delete_channel(self):
|
||||||
"""delete channel and all videos"""
|
"""delete channel and all videos"""
|
||||||
|
@ -317,6 +292,7 @@ class YoutubeChannel(YouTubeItem):
|
||||||
print(f"{self.youtube_id}: delete indexed videos")
|
print(f"{self.youtube_id}: delete indexed videos")
|
||||||
self.delete_es_videos()
|
self.delete_es_videos()
|
||||||
self.delete_es_comments()
|
self.delete_es_comments()
|
||||||
|
self.delete_es_subtitles()
|
||||||
self.del_in_es()
|
self.del_in_es()
|
||||||
|
|
||||||
def index_channel_playlists(self):
|
def index_channel_playlists(self):
|
||||||
|
@ -330,13 +306,12 @@ class YoutubeChannel(YouTubeItem):
|
||||||
print(f"{self.youtube_id}: no playlists found.")
|
print(f"{self.youtube_id}: no playlists found.")
|
||||||
return
|
return
|
||||||
|
|
||||||
all_youtube_ids = self.get_all_video_ids()
|
|
||||||
total = len(self.all_playlists)
|
total = len(self.all_playlists)
|
||||||
for idx, playlist in enumerate(self.all_playlists):
|
for idx, playlist in enumerate(self.all_playlists):
|
||||||
if self.task:
|
if self.task:
|
||||||
self._notify_single_playlist(idx, total)
|
self._notify_single_playlist(idx, total)
|
||||||
|
|
||||||
self._index_single_playlist(playlist, all_youtube_ids)
|
self._index_single_playlist(playlist)
|
||||||
print("add playlist: " + playlist[1])
|
print("add playlist: " + playlist[1])
|
||||||
|
|
||||||
def _notify_single_playlist(self, idx, total):
|
def _notify_single_playlist(self, idx, total):
|
||||||
|
@ -349,32 +324,10 @@ class YoutubeChannel(YouTubeItem):
|
||||||
self.task.send_progress(message, progress=(idx + 1) / total)
|
self.task.send_progress(message, progress=(idx + 1) / total)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _index_single_playlist(playlist, all_youtube_ids):
|
def _index_single_playlist(playlist):
|
||||||
"""add single playlist if needed"""
|
"""add single playlist if needed"""
|
||||||
playlist = YoutubePlaylist(playlist[0])
|
playlist = YoutubePlaylist(playlist[0])
|
||||||
playlist.all_youtube_ids = all_youtube_ids
|
playlist.update_playlist(skip_on_empty=True)
|
||||||
playlist.build_json()
|
|
||||||
if not playlist.json_data:
|
|
||||||
return
|
|
||||||
|
|
||||||
entries = playlist.json_data["playlist_entries"]
|
|
||||||
downloaded = [i for i in entries if i["downloaded"]]
|
|
||||||
if not downloaded:
|
|
||||||
return
|
|
||||||
|
|
||||||
playlist.upload_to_es()
|
|
||||||
playlist.add_vids_to_playlist()
|
|
||||||
playlist.get_playlist_art()
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def get_all_video_ids():
|
|
||||||
"""match all playlists with videos"""
|
|
||||||
handler = queue.PendingList()
|
|
||||||
handler.get_download()
|
|
||||||
handler.get_indexed()
|
|
||||||
all_youtube_ids = [i["youtube_id"] for i in handler.all_videos]
|
|
||||||
|
|
||||||
return all_youtube_ids
|
|
||||||
|
|
||||||
def get_channel_videos(self):
|
def get_channel_videos(self):
|
||||||
"""get all videos from channel"""
|
"""get all videos from channel"""
|
||||||
|
@ -411,9 +364,9 @@ class YoutubeChannel(YouTubeItem):
|
||||||
all_playlists = IndexPaginate("ta_playlist", data).get_results()
|
all_playlists = IndexPaginate("ta_playlist", data).get_results()
|
||||||
return all_playlists
|
return all_playlists
|
||||||
|
|
||||||
def get_overwrites(self):
|
def get_overwrites(self) -> dict:
|
||||||
"""get all per channel overwrites"""
|
"""get all per channel overwrites"""
|
||||||
return self.json_data.get("channel_overwrites", False)
|
return self.json_data.get("channel_overwrites", {})
|
||||||
|
|
||||||
def set_overwrites(self, overwrites):
|
def set_overwrites(self, overwrites):
|
||||||
"""set per channel overwrites"""
|
"""set per channel overwrites"""
|
||||||
|
|
|
@ -10,6 +10,7 @@ from datetime import datetime
|
||||||
from home.src.download.yt_dlp_base import YtWrap
|
from home.src.download.yt_dlp_base import YtWrap
|
||||||
from home.src.es.connect import ElasticWrap
|
from home.src.es.connect import ElasticWrap
|
||||||
from home.src.ta.config import AppConfig
|
from home.src.ta.config import AppConfig
|
||||||
|
from home.src.ta.ta_redis import RedisQueue
|
||||||
|
|
||||||
|
|
||||||
class Comments:
|
class Comments:
|
||||||
|
@ -63,10 +64,12 @@ class Comments:
|
||||||
"check_formats": None,
|
"check_formats": None,
|
||||||
"skip_download": True,
|
"skip_download": True,
|
||||||
"getcomments": True,
|
"getcomments": True,
|
||||||
|
"ignoreerrors": True,
|
||||||
"extractor_args": {
|
"extractor_args": {
|
||||||
"youtube": {
|
"youtube": {
|
||||||
"max_comments": max_comments_list,
|
"max_comments": max_comments_list,
|
||||||
"comment_sort": [comment_sort],
|
"comment_sort": [comment_sort],
|
||||||
|
"player_client": ["ios", "web"], # workaround yt-dlp #9554
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
@ -76,7 +79,7 @@ class Comments:
|
||||||
def get_yt_comments(self):
|
def get_yt_comments(self):
|
||||||
"""get comments from youtube"""
|
"""get comments from youtube"""
|
||||||
yt_obs = self.build_yt_obs()
|
yt_obs = self.build_yt_obs()
|
||||||
info_json = YtWrap(yt_obs).extract(self.youtube_id)
|
info_json = YtWrap(yt_obs, config=self.config).extract(self.youtube_id)
|
||||||
if not info_json:
|
if not info_json:
|
||||||
return False, False
|
return False, False
|
||||||
|
|
||||||
|
@ -114,17 +117,22 @@ class Comments:
|
||||||
|
|
||||||
time_text = time_text_datetime.strftime(format_string)
|
time_text = time_text_datetime.strftime(format_string)
|
||||||
|
|
||||||
|
if not comment.get("author"):
|
||||||
|
comment["author"] = comment.get("author_id", "Unknown")
|
||||||
|
|
||||||
cleaned_comment = {
|
cleaned_comment = {
|
||||||
"comment_id": comment["id"],
|
"comment_id": comment["id"],
|
||||||
"comment_text": comment["text"].replace("\xa0", ""),
|
"comment_text": comment["text"].replace("\xa0", ""),
|
||||||
"comment_timestamp": comment["timestamp"],
|
"comment_timestamp": comment["timestamp"],
|
||||||
"comment_time_text": time_text,
|
"comment_time_text": time_text,
|
||||||
"comment_likecount": comment["like_count"],
|
"comment_likecount": comment.get("like_count", None),
|
||||||
"comment_is_favorited": comment["is_favorited"],
|
"comment_is_favorited": comment.get("is_favorited", False),
|
||||||
"comment_author": comment["author"],
|
"comment_author": comment["author"],
|
||||||
"comment_author_id": comment["author_id"],
|
"comment_author_id": comment["author_id"],
|
||||||
"comment_author_thumbnail": comment["author_thumbnail"],
|
"comment_author_thumbnail": comment["author_thumbnail"],
|
||||||
"comment_author_is_uploader": comment["author_is_uploader"],
|
"comment_author_is_uploader": comment.get(
|
||||||
|
"author_is_uploader", False
|
||||||
|
),
|
||||||
"comment_parent": comment["parent"],
|
"comment_parent": comment["parent"],
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -182,20 +190,30 @@ class Comments:
|
||||||
class CommentList:
|
class CommentList:
|
||||||
"""interact with comments in group"""
|
"""interact with comments in group"""
|
||||||
|
|
||||||
def __init__(self, video_ids, task=False):
|
COMMENT_QUEUE = "index:comment"
|
||||||
self.video_ids = video_ids
|
|
||||||
|
def __init__(self, task=False):
|
||||||
self.task = task
|
self.task = task
|
||||||
self.config = AppConfig().config
|
self.config = AppConfig().config
|
||||||
|
|
||||||
def index(self):
|
def add(self, video_ids: list[str]) -> None:
|
||||||
"""index comments for list, init with task object to notify"""
|
"""add list of videos to get comments, if enabled in config"""
|
||||||
if not self.config["downloads"].get("comment_max"):
|
if not self.config["downloads"].get("comment_max"):
|
||||||
return
|
return
|
||||||
|
|
||||||
total_videos = len(self.video_ids)
|
RedisQueue(self.COMMENT_QUEUE).add_list(video_ids)
|
||||||
for idx, youtube_id in enumerate(self.video_ids):
|
|
||||||
|
def index(self):
|
||||||
|
"""run comment index"""
|
||||||
|
queue = RedisQueue(self.COMMENT_QUEUE)
|
||||||
|
while True:
|
||||||
|
total = queue.max_score()
|
||||||
|
youtube_id, idx = queue.get_next()
|
||||||
|
if not youtube_id or not idx or not total:
|
||||||
|
break
|
||||||
|
|
||||||
if self.task:
|
if self.task:
|
||||||
self.notify(idx, total_videos)
|
self.notify(idx, total)
|
||||||
|
|
||||||
comment = Comments(youtube_id, config=self.config)
|
comment = Comments(youtube_id, config=self.config)
|
||||||
comment.build_json()
|
comment.build_json()
|
||||||
|
@ -204,6 +222,6 @@ class CommentList:
|
||||||
|
|
||||||
def notify(self, idx, total_videos):
|
def notify(self, idx, total_videos):
|
||||||
"""send notification on task"""
|
"""send notification on task"""
|
||||||
message = [f"Add comments for new videos {idx + 1}/{total_videos}"]
|
message = [f"Add comments for new videos {idx}/{total_videos}"]
|
||||||
progress = (idx + 1) / total_videos
|
progress = idx / total_videos
|
||||||
self.task.send_progress(message, progress=progress)
|
self.task.send_progress(message, progress=progress)
|
||||||
|
|
|
@ -1,201 +1,126 @@
|
||||||
"""
|
"""
|
||||||
Functionality:
|
Functionality:
|
||||||
- reindexing old documents
|
|
||||||
- syncing updated values between indexes
|
|
||||||
- scan the filesystem to delete or index
|
- scan the filesystem to delete or index
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import json
|
|
||||||
import os
|
import os
|
||||||
|
|
||||||
from home.src.download.queue import PendingList
|
from home.src.es.connect import ElasticWrap, IndexPaginate
|
||||||
from home.src.es.connect import ElasticWrap
|
|
||||||
from home.src.index.comments import CommentList
|
from home.src.index.comments import CommentList
|
||||||
from home.src.index.video import index_new_video
|
from home.src.index.video import YoutubeVideo, index_new_video
|
||||||
from home.src.ta.config import AppConfig
|
from home.src.ta.helper import ignore_filelist
|
||||||
from home.src.ta.helper import clean_string, ignore_filelist
|
from home.src.ta.settings import EnvironmentSettings
|
||||||
from PIL import ImageFile
|
|
||||||
|
|
||||||
ImageFile.LOAD_TRUNCATED_IMAGES = True
|
|
||||||
|
|
||||||
|
|
||||||
class ScannerBase:
|
class Scanner:
|
||||||
"""scan the filesystem base class"""
|
"""scan index and filesystem"""
|
||||||
|
|
||||||
CONFIG = AppConfig().config
|
VIDEOS: str = EnvironmentSettings.MEDIA_DIR
|
||||||
VIDEOS = CONFIG["application"]["videos"]
|
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self, task=False) -> None:
|
||||||
self.to_index = False
|
|
||||||
self.to_delete = False
|
|
||||||
self.mismatch = False
|
|
||||||
self.to_rename = False
|
|
||||||
|
|
||||||
def scan(self):
|
|
||||||
"""entry point, scan and compare"""
|
|
||||||
all_downloaded = self._get_all_downloaded()
|
|
||||||
all_indexed = self._get_all_indexed()
|
|
||||||
self.list_comarison(all_downloaded, all_indexed)
|
|
||||||
|
|
||||||
def _get_all_downloaded(self):
|
|
||||||
"""get a list of all video files downloaded"""
|
|
||||||
channels = os.listdir(self.VIDEOS)
|
|
||||||
all_channels = ignore_filelist(channels)
|
|
||||||
all_channels.sort()
|
|
||||||
all_downloaded = []
|
|
||||||
for channel_name in all_channels:
|
|
||||||
channel_path = os.path.join(self.VIDEOS, channel_name)
|
|
||||||
channel_files = os.listdir(channel_path)
|
|
||||||
channel_files_clean = ignore_filelist(channel_files)
|
|
||||||
all_videos = [i for i in channel_files_clean if i.endswith(".mp4")]
|
|
||||||
for video in all_videos:
|
|
||||||
youtube_id = video[9:20]
|
|
||||||
all_downloaded.append((channel_name, video, youtube_id))
|
|
||||||
|
|
||||||
return all_downloaded
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _get_all_indexed():
|
|
||||||
"""get a list of all indexed videos"""
|
|
||||||
index_handler = PendingList()
|
|
||||||
index_handler.get_download()
|
|
||||||
index_handler.get_indexed()
|
|
||||||
|
|
||||||
all_indexed = []
|
|
||||||
for video in index_handler.all_videos:
|
|
||||||
youtube_id = video["youtube_id"]
|
|
||||||
media_url = video["media_url"]
|
|
||||||
published = video["published"]
|
|
||||||
title = video["title"]
|
|
||||||
all_indexed.append((youtube_id, media_url, published, title))
|
|
||||||
return all_indexed
|
|
||||||
|
|
||||||
def list_comarison(self, all_downloaded, all_indexed):
|
|
||||||
"""compare the lists to figure out what to do"""
|
|
||||||
self._find_unindexed(all_downloaded, all_indexed)
|
|
||||||
self._find_missing(all_downloaded, all_indexed)
|
|
||||||
self._find_bad_media_url(all_downloaded, all_indexed)
|
|
||||||
|
|
||||||
def _find_unindexed(self, all_downloaded, all_indexed):
|
|
||||||
"""find video files without a matching document indexed"""
|
|
||||||
all_indexed_ids = [i[0] for i in all_indexed]
|
|
||||||
self.to_index = []
|
|
||||||
for downloaded in all_downloaded:
|
|
||||||
if downloaded[2] not in all_indexed_ids:
|
|
||||||
self.to_index.append(downloaded)
|
|
||||||
|
|
||||||
def _find_missing(self, all_downloaded, all_indexed):
|
|
||||||
"""find indexed videos without matching media file"""
|
|
||||||
all_downloaded_ids = [i[2] for i in all_downloaded]
|
|
||||||
self.to_delete = []
|
|
||||||
for video in all_indexed:
|
|
||||||
youtube_id = video[0]
|
|
||||||
if youtube_id not in all_downloaded_ids:
|
|
||||||
self.to_delete.append(video)
|
|
||||||
|
|
||||||
def _find_bad_media_url(self, all_downloaded, all_indexed):
|
|
||||||
"""rename media files not matching the indexed title"""
|
|
||||||
self.mismatch = []
|
|
||||||
self.to_rename = []
|
|
||||||
|
|
||||||
for downloaded in all_downloaded:
|
|
||||||
channel, filename, downloaded_id = downloaded
|
|
||||||
# find in indexed
|
|
||||||
for indexed in all_indexed:
|
|
||||||
indexed_id, media_url, published, title = indexed
|
|
||||||
if indexed_id == downloaded_id:
|
|
||||||
# found it
|
|
||||||
pub = published.replace("-", "")
|
|
||||||
expected = f"{pub}_{indexed_id}_{clean_string(title)}.mp4"
|
|
||||||
new_url = os.path.join(channel, expected)
|
|
||||||
if expected != filename:
|
|
||||||
# file to rename
|
|
||||||
self.to_rename.append((channel, filename, expected))
|
|
||||||
if media_url != new_url:
|
|
||||||
# media_url to update in es
|
|
||||||
self.mismatch.append((indexed_id, new_url))
|
|
||||||
|
|
||||||
break
|
|
||||||
|
|
||||||
|
|
||||||
class Filesystem(ScannerBase):
|
|
||||||
"""handle scanning and fixing from filesystem"""
|
|
||||||
|
|
||||||
def __init__(self, task=False):
|
|
||||||
super().__init__()
|
|
||||||
self.task = task
|
self.task = task
|
||||||
|
self.to_delete: set[str] = set()
|
||||||
|
self.to_index: set[str] = set()
|
||||||
|
|
||||||
def process(self):
|
def scan(self) -> None:
|
||||||
"""entry point"""
|
"""scan the filesystem"""
|
||||||
self.task.send_progress(["Scanning your archive and index."])
|
downloaded: set[str] = self._get_downloaded()
|
||||||
self.scan()
|
indexed: set[str] = self._get_indexed()
|
||||||
self.rename_files()
|
self.to_index = downloaded - indexed
|
||||||
self.send_mismatch_bulk()
|
self.to_delete = indexed - downloaded
|
||||||
self.delete_from_index()
|
|
||||||
self.add_missing()
|
|
||||||
|
|
||||||
def rename_files(self):
|
def _get_downloaded(self) -> set[str]:
|
||||||
"""rename media files as identified by find_bad_media_url"""
|
"""get downloaded ids"""
|
||||||
if not self.to_rename:
|
if self.task:
|
||||||
return
|
self.task.send_progress(["Scan your filesystem for videos."])
|
||||||
|
|
||||||
total = len(self.to_rename)
|
downloaded: set = set()
|
||||||
self.task.send_progress([f"Rename {total} media files."])
|
channels = ignore_filelist(os.listdir(self.VIDEOS))
|
||||||
for bad_filename in self.to_rename:
|
for channel in channels:
|
||||||
channel, filename, expected_filename = bad_filename
|
folder = os.path.join(self.VIDEOS, channel)
|
||||||
print(f"renaming [{filename}] to [{expected_filename}]")
|
files = ignore_filelist(os.listdir(folder))
|
||||||
old_path = os.path.join(self.VIDEOS, channel, filename)
|
downloaded.update({i.split(".")[0] for i in files})
|
||||||
new_path = os.path.join(self.VIDEOS, channel, expected_filename)
|
|
||||||
os.rename(old_path, new_path)
|
|
||||||
|
|
||||||
def send_mismatch_bulk(self):
|
return downloaded
|
||||||
"""build bulk update"""
|
|
||||||
if not self.mismatch:
|
|
||||||
return
|
|
||||||
|
|
||||||
total = len(self.mismatch)
|
def _get_indexed(self) -> set:
|
||||||
self.task.send_progress([f"Fix media urls for {total} files"])
|
"""get all indexed ids"""
|
||||||
bulk_list = []
|
if self.task:
|
||||||
for video_mismatch in self.mismatch:
|
self.task.send_progress(["Get all videos indexed."])
|
||||||
youtube_id, media_url = video_mismatch
|
|
||||||
print(f"{youtube_id}: fixing media url {media_url}")
|
|
||||||
action = {"update": {"_id": youtube_id, "_index": "ta_video"}}
|
|
||||||
source = {"doc": {"media_url": media_url}}
|
|
||||||
bulk_list.append(json.dumps(action))
|
|
||||||
bulk_list.append(json.dumps(source))
|
|
||||||
# add last newline
|
|
||||||
bulk_list.append("\n")
|
|
||||||
data = "\n".join(bulk_list)
|
|
||||||
_, _ = ElasticWrap("_bulk").post(data=data, ndjson=True)
|
|
||||||
|
|
||||||
def delete_from_index(self):
|
data = {"query": {"match_all": {}}, "_source": ["youtube_id"]}
|
||||||
"""find indexed but deleted mediafile"""
|
response = IndexPaginate("ta_video", data).get_results()
|
||||||
|
return {i["youtube_id"] for i in response}
|
||||||
|
|
||||||
|
def apply(self) -> None:
|
||||||
|
"""apply all changes"""
|
||||||
|
self.delete()
|
||||||
|
self.index()
|
||||||
|
self.url_fix()
|
||||||
|
|
||||||
|
def delete(self) -> None:
|
||||||
|
"""delete videos from index"""
|
||||||
if not self.to_delete:
|
if not self.to_delete:
|
||||||
|
print("nothing to delete")
|
||||||
return
|
return
|
||||||
|
|
||||||
total = len(self.to_delete)
|
if self.task:
|
||||||
self.task.send_progress([f"Clean up {total} items from index."])
|
self.task.send_progress(
|
||||||
for indexed in self.to_delete:
|
[f"Remove {len(self.to_delete)} videos from index."]
|
||||||
youtube_id = indexed[0]
|
)
|
||||||
print(f"deleting {youtube_id} from index")
|
|
||||||
path = f"ta_video/_doc/{youtube_id}"
|
|
||||||
_, _ = ElasticWrap(path).delete()
|
|
||||||
|
|
||||||
def add_missing(self):
|
for youtube_id in self.to_delete:
|
||||||
"""add missing videos to index"""
|
YoutubeVideo(youtube_id).delete_media_file()
|
||||||
video_ids = [i[2] for i in self.to_index]
|
|
||||||
if not video_ids:
|
def index(self) -> None:
|
||||||
|
"""index new"""
|
||||||
|
if not self.to_index:
|
||||||
|
print("nothing to index")
|
||||||
return
|
return
|
||||||
|
|
||||||
total = len(video_ids)
|
total = len(self.to_index)
|
||||||
for idx, youtube_id in enumerate(video_ids):
|
for idx, youtube_id in enumerate(self.to_index):
|
||||||
if self.task:
|
if self.task:
|
||||||
self.task.send_progress(
|
self.task.send_progress(
|
||||||
message_lines=[
|
message_lines=[
|
||||||
f"Index missing video {youtube_id}, {idx}/{total}"
|
f"Index missing video {youtube_id}, {idx + 1}/{total}"
|
||||||
],
|
],
|
||||||
progress=(idx + 1) / total,
|
progress=(idx + 1) / total,
|
||||||
)
|
)
|
||||||
index_new_video(youtube_id)
|
index_new_video(youtube_id)
|
||||||
|
|
||||||
CommentList(video_ids, task=self.task).index()
|
comment_list = CommentList(task=self.task)
|
||||||
|
comment_list.add(video_ids=list(self.to_index))
|
||||||
|
comment_list.index()
|
||||||
|
|
||||||
|
def url_fix(self) -> None:
|
||||||
|
"""
|
||||||
|
update path v0.3.6 to v0.3.7
|
||||||
|
fix url not matching channel-videoid pattern
|
||||||
|
"""
|
||||||
|
bool_must = (
|
||||||
|
"doc['media_url'].value == "
|
||||||
|
+ "(doc['channel.channel_id'].value + '/' + "
|
||||||
|
+ "doc['youtube_id'].value) + '.mp4'"
|
||||||
|
)
|
||||||
|
to_update = (
|
||||||
|
"ctx._source['media_url'] = "
|
||||||
|
+ "ctx._source.channel['channel_id'] + '/' + "
|
||||||
|
+ "ctx._source['youtube_id'] + '.mp4'"
|
||||||
|
)
|
||||||
|
data = {
|
||||||
|
"query": {
|
||||||
|
"bool": {
|
||||||
|
"must_not": [{"script": {"script": {"source": bool_must}}}]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"script": {"source": to_update},
|
||||||
|
}
|
||||||
|
response, _ = ElasticWrap("ta_video/_update_by_query").post(data=data)
|
||||||
|
updated = response.get("updates")
|
||||||
|
if updated:
|
||||||
|
print(f"updated {updated} bad media_url")
|
||||||
|
if self.task:
|
||||||
|
self.task.send_progress(
|
||||||
|
[f"Updated {updated} wrong media urls."]
|
||||||
|
)
|
||||||
|
|
|
@ -8,34 +8,42 @@ import math
|
||||||
from home.src.download.yt_dlp_base import YtWrap
|
from home.src.download.yt_dlp_base import YtWrap
|
||||||
from home.src.es.connect import ElasticWrap
|
from home.src.es.connect import ElasticWrap
|
||||||
from home.src.ta.config import AppConfig
|
from home.src.ta.config import AppConfig
|
||||||
from home.src.ta.ta_redis import RedisArchivist
|
from home.src.ta.users import UserConfig
|
||||||
|
|
||||||
|
|
||||||
class YouTubeItem:
|
class YouTubeItem:
|
||||||
"""base class for youtube"""
|
"""base class for youtube"""
|
||||||
|
|
||||||
es_path = False
|
es_path = False
|
||||||
index_name = False
|
index_name = ""
|
||||||
yt_base = False
|
yt_base = ""
|
||||||
yt_obs = {
|
yt_obs: dict[str, bool | str] = {
|
||||||
"skip_download": True,
|
"skip_download": True,
|
||||||
"noplaylist": True,
|
"noplaylist": True,
|
||||||
}
|
}
|
||||||
|
|
||||||
def __init__(self, youtube_id):
|
def __init__(self, youtube_id):
|
||||||
self.youtube_id = youtube_id
|
self.youtube_id = youtube_id
|
||||||
|
self.es_path = f"{self.index_name}/_doc/{youtube_id}"
|
||||||
self.config = AppConfig().config
|
self.config = AppConfig().config
|
||||||
self.app_conf = self.config["application"]
|
|
||||||
self.youtube_meta = False
|
self.youtube_meta = False
|
||||||
self.json_data = False
|
self.json_data = False
|
||||||
|
|
||||||
|
def build_yt_url(self):
|
||||||
|
"""build youtube url"""
|
||||||
|
return self.yt_base + self.youtube_id
|
||||||
|
|
||||||
def get_from_youtube(self):
|
def get_from_youtube(self):
|
||||||
"""use yt-dlp to get meta data from youtube"""
|
"""use yt-dlp to get meta data from youtube"""
|
||||||
print(f"{self.youtube_id}: get metadata from youtube")
|
print(f"{self.youtube_id}: get metadata from youtube")
|
||||||
url = self.yt_base + self.youtube_id
|
obs_request = self.yt_obs.copy()
|
||||||
response = YtWrap(self.yt_obs, self.config).extract(url)
|
if self.config["downloads"]["extractor_lang"]:
|
||||||
|
langs = self.config["downloads"]["extractor_lang"]
|
||||||
|
langs_list = [i.strip() for i in langs.split(",")]
|
||||||
|
obs_request["extractor_args"] = {"youtube": {"lang": langs_list}}
|
||||||
|
|
||||||
self.youtube_meta = response
|
url = self.build_yt_url()
|
||||||
|
self.youtube_meta = YtWrap(obs_request, self.config).extract(url)
|
||||||
|
|
||||||
def get_from_es(self):
|
def get_from_es(self):
|
||||||
"""get indexed data from elastic search"""
|
"""get indexed data from elastic search"""
|
||||||
|
@ -91,13 +99,7 @@ class Pagination:
|
||||||
|
|
||||||
def get_page_size(self):
|
def get_page_size(self):
|
||||||
"""get default or user modified page_size"""
|
"""get default or user modified page_size"""
|
||||||
key = f"{self.request.user.id}:page_size"
|
return UserConfig(self.request.user.id).get_value("page_size")
|
||||||
page_size = RedisArchivist().get_message(key)["status"]
|
|
||||||
if not page_size:
|
|
||||||
config = AppConfig().config
|
|
||||||
page_size = config["archive"]["page_size"]
|
|
||||||
|
|
||||||
return page_size
|
|
||||||
|
|
||||||
def first_guess(self):
|
def first_guess(self):
|
||||||
"""build first guess before api call"""
|
"""build first guess before api call"""
|
||||||
|
|
|
@ -16,6 +16,7 @@ from home.src.index.comments import CommentList
|
||||||
from home.src.index.video import YoutubeVideo
|
from home.src.index.video import YoutubeVideo
|
||||||
from home.src.ta.config import AppConfig
|
from home.src.ta.config import AppConfig
|
||||||
from home.src.ta.helper import ignore_filelist
|
from home.src.ta.helper import ignore_filelist
|
||||||
|
from home.src.ta.settings import EnvironmentSettings
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
from yt_dlp.utils import ISO639Utils
|
from yt_dlp.utils import ISO639Utils
|
||||||
|
|
||||||
|
@ -28,9 +29,12 @@ class ImportFolderScanner:
|
||||||
"""
|
"""
|
||||||
|
|
||||||
CONFIG = AppConfig().config
|
CONFIG = AppConfig().config
|
||||||
CACHE_DIR = CONFIG["application"]["cache_dir"]
|
CACHE_DIR = EnvironmentSettings.CACHE_DIR
|
||||||
IMPORT_DIR = os.path.join(CACHE_DIR, "import")
|
IMPORT_DIR = os.path.join(CACHE_DIR, "import")
|
||||||
|
|
||||||
|
"""All extensions should be in lowercase until better handling is in place.
|
||||||
|
Described in Issue #502.
|
||||||
|
"""
|
||||||
EXT_MAP = {
|
EXT_MAP = {
|
||||||
"media": [".mp4", ".mkv", ".webm"],
|
"media": [".mp4", ".mkv", ".webm"],
|
||||||
"metadata": [".json"],
|
"metadata": [".json"],
|
||||||
|
@ -118,7 +122,7 @@ class ImportFolderScanner:
|
||||||
"""detect metadata type for file"""
|
"""detect metadata type for file"""
|
||||||
|
|
||||||
for key, value in self.EXT_MAP.items():
|
for key, value in self.EXT_MAP.items():
|
||||||
if ext in value:
|
if ext.lower() in value:
|
||||||
return key, file_path
|
return key, file_path
|
||||||
|
|
||||||
return False, False
|
return False, False
|
||||||
|
@ -143,7 +147,9 @@ class ImportFolderScanner:
|
||||||
ManualImport(current_video, self.CONFIG).run()
|
ManualImport(current_video, self.CONFIG).run()
|
||||||
|
|
||||||
video_ids = [i["video_id"] for i in self.to_import]
|
video_ids = [i["video_id"] for i in self.to_import]
|
||||||
CommentList(video_ids, task=self.task).index()
|
comment_list = CommentList(task=self.task)
|
||||||
|
comment_list.add(video_ids=video_ids)
|
||||||
|
comment_list.index()
|
||||||
|
|
||||||
def _notify(self, idx, current_video):
|
def _notify(self, idx, current_video):
|
||||||
"""send notification back to task"""
|
"""send notification back to task"""
|
||||||
|
@ -430,9 +436,9 @@ class ManualImport:
|
||||||
|
|
||||||
def _move_to_archive(self, json_data):
|
def _move_to_archive(self, json_data):
|
||||||
"""move identified media file to archive"""
|
"""move identified media file to archive"""
|
||||||
videos = self.config["application"]["videos"]
|
videos = EnvironmentSettings.MEDIA_DIR
|
||||||
host_uid = self.config["application"]["HOST_UID"]
|
host_uid = EnvironmentSettings.HOST_UID
|
||||||
host_gid = self.config["application"]["HOST_GID"]
|
host_gid = EnvironmentSettings.HOST_GID
|
||||||
|
|
||||||
channel, file = os.path.split(json_data["media_url"])
|
channel, file = os.path.split(json_data["media_url"])
|
||||||
channel_folder = os.path.join(videos, channel)
|
channel_folder = os.path.join(videos, channel)
|
||||||
|
@ -469,7 +475,7 @@ class ManualImport:
|
||||||
os.remove(subtitle_file)
|
os.remove(subtitle_file)
|
||||||
|
|
||||||
channel_info = os.path.join(
|
channel_info = os.path.join(
|
||||||
self.config["application"]["cache_dir"],
|
EnvironmentSettings.CACHE_DIR,
|
||||||
"import",
|
"import",
|
||||||
f"{json_data['channel']['channel_id']}.info.json",
|
f"{json_data['channel']['channel_id']}.info.json",
|
||||||
)
|
)
|
||||||
|
|
|
@ -8,7 +8,8 @@ import json
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
from home.src.download.thumbnails import ThumbManager
|
from home.src.download.thumbnails import ThumbManager
|
||||||
from home.src.es.connect import ElasticWrap
|
from home.src.es.connect import ElasticWrap, IndexPaginate
|
||||||
|
from home.src.index import channel
|
||||||
from home.src.index.generic import YouTubeItem
|
from home.src.index.generic import YouTubeItem
|
||||||
from home.src.index.video import YoutubeVideo
|
from home.src.index.video import YoutubeVideo
|
||||||
|
|
||||||
|
@ -26,10 +27,8 @@ class YoutubePlaylist(YouTubeItem):
|
||||||
|
|
||||||
def __init__(self, youtube_id):
|
def __init__(self, youtube_id):
|
||||||
super().__init__(youtube_id)
|
super().__init__(youtube_id)
|
||||||
self.es_path = f"{self.index_name}/_doc/{youtube_id}"
|
|
||||||
self.all_members = False
|
self.all_members = False
|
||||||
self.nav = False
|
self.nav = False
|
||||||
self.all_youtube_ids = []
|
|
||||||
|
|
||||||
def build_json(self, scrape=False):
|
def build_json(self, scrape=False):
|
||||||
"""collection to create json_data"""
|
"""collection to create json_data"""
|
||||||
|
@ -46,7 +45,9 @@ class YoutubePlaylist(YouTubeItem):
|
||||||
return
|
return
|
||||||
|
|
||||||
self.process_youtube_meta()
|
self.process_youtube_meta()
|
||||||
self.get_entries()
|
self._ensure_channel()
|
||||||
|
ids_found = self.get_local_vids()
|
||||||
|
self.get_entries(ids_found)
|
||||||
self.json_data["playlist_entries"] = self.all_members
|
self.json_data["playlist_entries"] = self.all_members
|
||||||
self.json_data["playlist_subscribed"] = subscribed
|
self.json_data["playlist_subscribed"] = subscribed
|
||||||
|
|
||||||
|
@ -67,27 +68,40 @@ class YoutubePlaylist(YouTubeItem):
|
||||||
"playlist_thumbnail": playlist_thumbnail,
|
"playlist_thumbnail": playlist_thumbnail,
|
||||||
"playlist_description": self.youtube_meta["description"] or False,
|
"playlist_description": self.youtube_meta["description"] or False,
|
||||||
"playlist_last_refresh": int(datetime.now().timestamp()),
|
"playlist_last_refresh": int(datetime.now().timestamp()),
|
||||||
|
"playlist_type": "regular",
|
||||||
}
|
}
|
||||||
|
|
||||||
def get_entries(self, playlistend=False):
|
def _ensure_channel(self):
|
||||||
"""get all videos in playlist"""
|
"""make sure channel is indexed"""
|
||||||
if playlistend:
|
channel_id = self.json_data["playlist_channel_id"]
|
||||||
# implement playlist end
|
channel_handler = channel.YoutubeChannel(channel_id)
|
||||||
print(playlistend)
|
channel_handler.build_json(upload=True)
|
||||||
|
|
||||||
|
def get_local_vids(self) -> list[str]:
|
||||||
|
"""get local video ids from youtube entries"""
|
||||||
|
entries = self.youtube_meta["entries"]
|
||||||
|
data = {
|
||||||
|
"query": {"terms": {"youtube_id": [i["id"] for i in entries]}},
|
||||||
|
"_source": ["youtube_id"],
|
||||||
|
}
|
||||||
|
indexed_vids = IndexPaginate("ta_video", data).get_results()
|
||||||
|
ids_found = [i["youtube_id"] for i in indexed_vids]
|
||||||
|
|
||||||
|
return ids_found
|
||||||
|
|
||||||
|
def get_entries(self, ids_found) -> None:
|
||||||
|
"""get all videos in playlist, match downloaded with ids_found"""
|
||||||
all_members = []
|
all_members = []
|
||||||
for idx, entry in enumerate(self.youtube_meta["entries"]):
|
for idx, entry in enumerate(self.youtube_meta["entries"]):
|
||||||
if self.all_youtube_ids:
|
|
||||||
downloaded = entry["id"] in self.all_youtube_ids
|
|
||||||
else:
|
|
||||||
downloaded = False
|
|
||||||
if not entry["channel"]:
|
if not entry["channel"]:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
to_append = {
|
to_append = {
|
||||||
"youtube_id": entry["id"],
|
"youtube_id": entry["id"],
|
||||||
"title": entry["title"],
|
"title": entry["title"],
|
||||||
"uploader": entry["channel"],
|
"uploader": entry["channel"],
|
||||||
"idx": idx,
|
"idx": idx,
|
||||||
"downloaded": downloaded,
|
"downloaded": entry["id"] in ids_found,
|
||||||
}
|
}
|
||||||
all_members.append(to_append)
|
all_members.append(to_append)
|
||||||
|
|
||||||
|
@ -128,17 +142,50 @@ class YoutubePlaylist(YouTubeItem):
|
||||||
|
|
||||||
ElasticWrap("_bulk").post(query_str, ndjson=True)
|
ElasticWrap("_bulk").post(query_str, ndjson=True)
|
||||||
|
|
||||||
def update_playlist(self):
|
def remove_vids_from_playlist(self):
|
||||||
|
"""remove playlist ids from videos if needed"""
|
||||||
|
needed = [i["youtube_id"] for i in self.json_data["playlist_entries"]]
|
||||||
|
data = {
|
||||||
|
"query": {"match": {"playlist": self.youtube_id}},
|
||||||
|
"_source": ["youtube_id"],
|
||||||
|
}
|
||||||
|
result = IndexPaginate("ta_video", data).get_results()
|
||||||
|
to_remove = [
|
||||||
|
i["youtube_id"] for i in result if i["youtube_id"] not in needed
|
||||||
|
]
|
||||||
|
s = "ctx._source.playlist.removeAll(Collections.singleton(params.rm))"
|
||||||
|
for video_id in to_remove:
|
||||||
|
query = {
|
||||||
|
"script": {
|
||||||
|
"source": s,
|
||||||
|
"lang": "painless",
|
||||||
|
"params": {"rm": self.youtube_id},
|
||||||
|
},
|
||||||
|
"query": {"match": {"youtube_id": video_id}},
|
||||||
|
}
|
||||||
|
path = "ta_video/_update_by_query"
|
||||||
|
_, status_code = ElasticWrap(path).post(query)
|
||||||
|
if status_code == 200:
|
||||||
|
print(f"{self.youtube_id}: removed {video_id} from playlist")
|
||||||
|
|
||||||
|
def update_playlist(self, skip_on_empty=False):
|
||||||
"""update metadata for playlist with data from YouTube"""
|
"""update metadata for playlist with data from YouTube"""
|
||||||
self.get_from_es()
|
self.build_json(scrape=True)
|
||||||
subscribed = self.json_data["playlist_subscribed"]
|
|
||||||
self.get_from_youtube()
|
|
||||||
if not self.json_data:
|
if not self.json_data:
|
||||||
# return false to deactivate
|
# return false to deactivate
|
||||||
return False
|
return False
|
||||||
|
|
||||||
self.json_data["playlist_subscribed"] = subscribed
|
if skip_on_empty:
|
||||||
|
has_item_downloaded = any(
|
||||||
|
i["downloaded"] for i in self.json_data["playlist_entries"]
|
||||||
|
)
|
||||||
|
if not has_item_downloaded:
|
||||||
|
return True
|
||||||
|
|
||||||
self.upload_to_es()
|
self.upload_to_es()
|
||||||
|
self.add_vids_to_playlist()
|
||||||
|
self.remove_vids_from_playlist()
|
||||||
|
self.get_playlist_art()
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def build_nav(self, youtube_id):
|
def build_nav(self, youtube_id):
|
||||||
|
@ -179,6 +226,7 @@ class YoutubePlaylist(YouTubeItem):
|
||||||
|
|
||||||
def delete_metadata(self):
|
def delete_metadata(self):
|
||||||
"""delete metadata for playlist"""
|
"""delete metadata for playlist"""
|
||||||
|
self.delete_videos_metadata()
|
||||||
script = (
|
script = (
|
||||||
"ctx._source.playlist.removeAll("
|
"ctx._source.playlist.removeAll("
|
||||||
+ "Collections.singleton(params.playlist)) "
|
+ "Collections.singleton(params.playlist)) "
|
||||||
|
@ -196,6 +244,30 @@ class YoutubePlaylist(YouTubeItem):
|
||||||
_, _ = ElasticWrap("ta_video/_update_by_query").post(data)
|
_, _ = ElasticWrap("ta_video/_update_by_query").post(data)
|
||||||
self.del_in_es()
|
self.del_in_es()
|
||||||
|
|
||||||
|
def is_custom_playlist(self):
|
||||||
|
self.get_from_es()
|
||||||
|
return self.json_data["playlist_type"] == "custom"
|
||||||
|
|
||||||
|
def delete_videos_metadata(self, channel_id=None):
|
||||||
|
"""delete video metadata for a specific channel"""
|
||||||
|
self.get_from_es()
|
||||||
|
playlist = self.json_data["playlist_entries"]
|
||||||
|
i = 0
|
||||||
|
while i < len(playlist):
|
||||||
|
video_id = playlist[i]["youtube_id"]
|
||||||
|
video = YoutubeVideo(video_id)
|
||||||
|
video.get_from_es()
|
||||||
|
if (
|
||||||
|
channel_id is None
|
||||||
|
or video.json_data["channel"]["channel_id"] == channel_id
|
||||||
|
):
|
||||||
|
playlist.pop(i)
|
||||||
|
self.remove_playlist_from_video(video_id)
|
||||||
|
i -= 1
|
||||||
|
i += 1
|
||||||
|
self.set_playlist_thumbnail()
|
||||||
|
self.upload_to_es()
|
||||||
|
|
||||||
def delete_videos_playlist(self):
|
def delete_videos_playlist(self):
|
||||||
"""delete playlist with all videos"""
|
"""delete playlist with all videos"""
|
||||||
print(f"{self.youtube_id}: delete playlist")
|
print(f"{self.youtube_id}: delete playlist")
|
||||||
|
@ -209,3 +281,159 @@ class YoutubePlaylist(YouTubeItem):
|
||||||
YoutubeVideo(youtube_id).delete_media_file()
|
YoutubeVideo(youtube_id).delete_media_file()
|
||||||
|
|
||||||
self.delete_metadata()
|
self.delete_metadata()
|
||||||
|
|
||||||
|
def create(self, name):
|
||||||
|
self.json_data = {
|
||||||
|
"playlist_id": self.youtube_id,
|
||||||
|
"playlist_active": False,
|
||||||
|
"playlist_name": name,
|
||||||
|
"playlist_last_refresh": int(datetime.now().timestamp()),
|
||||||
|
"playlist_entries": [],
|
||||||
|
"playlist_type": "custom",
|
||||||
|
"playlist_channel": None,
|
||||||
|
"playlist_channel_id": None,
|
||||||
|
"playlist_description": False,
|
||||||
|
"playlist_thumbnail": False,
|
||||||
|
"playlist_subscribed": False,
|
||||||
|
}
|
||||||
|
self.upload_to_es()
|
||||||
|
self.get_playlist_art()
|
||||||
|
return True
|
||||||
|
|
||||||
|
def add_video_to_playlist(self, video_id):
|
||||||
|
self.get_from_es()
|
||||||
|
video_metadata = self.get_video_metadata(video_id)
|
||||||
|
video_metadata["idx"] = len(self.json_data["playlist_entries"])
|
||||||
|
|
||||||
|
if not self.playlist_entries_contains(video_id):
|
||||||
|
self.json_data["playlist_entries"].append(video_metadata)
|
||||||
|
self.json_data["playlist_last_refresh"] = int(
|
||||||
|
datetime.now().timestamp()
|
||||||
|
)
|
||||||
|
self.set_playlist_thumbnail()
|
||||||
|
self.upload_to_es()
|
||||||
|
video = YoutubeVideo(video_id)
|
||||||
|
video.get_from_es()
|
||||||
|
if "playlist" not in video.json_data:
|
||||||
|
video.json_data["playlist"] = []
|
||||||
|
video.json_data["playlist"].append(self.youtube_id)
|
||||||
|
video.upload_to_es()
|
||||||
|
return True
|
||||||
|
|
||||||
|
def remove_playlist_from_video(self, video_id):
|
||||||
|
video = YoutubeVideo(video_id)
|
||||||
|
video.get_from_es()
|
||||||
|
if video.json_data is not None and "playlist" in video.json_data:
|
||||||
|
video.json_data["playlist"].remove(self.youtube_id)
|
||||||
|
video.upload_to_es()
|
||||||
|
|
||||||
|
def move_video(self, video_id, action, hide_watched=False):
|
||||||
|
self.get_from_es()
|
||||||
|
video_index = self.get_video_index(video_id)
|
||||||
|
playlist = self.json_data["playlist_entries"]
|
||||||
|
item = playlist[video_index]
|
||||||
|
playlist.pop(video_index)
|
||||||
|
if action == "remove":
|
||||||
|
self.remove_playlist_from_video(item["youtube_id"])
|
||||||
|
else:
|
||||||
|
if action == "up":
|
||||||
|
while True:
|
||||||
|
video_index = max(0, video_index - 1)
|
||||||
|
if (
|
||||||
|
not hide_watched
|
||||||
|
or video_index == 0
|
||||||
|
or (
|
||||||
|
not self.get_video_is_watched(
|
||||||
|
playlist[video_index]["youtube_id"]
|
||||||
|
)
|
||||||
|
)
|
||||||
|
):
|
||||||
|
break
|
||||||
|
elif action == "down":
|
||||||
|
while True:
|
||||||
|
video_index = min(len(playlist), video_index + 1)
|
||||||
|
if (
|
||||||
|
not hide_watched
|
||||||
|
or video_index == len(playlist)
|
||||||
|
or (
|
||||||
|
not self.get_video_is_watched(
|
||||||
|
playlist[video_index - 1]["youtube_id"]
|
||||||
|
)
|
||||||
|
)
|
||||||
|
):
|
||||||
|
break
|
||||||
|
elif action == "top":
|
||||||
|
video_index = 0
|
||||||
|
else:
|
||||||
|
video_index = len(playlist)
|
||||||
|
playlist.insert(video_index, item)
|
||||||
|
self.json_data["playlist_last_refresh"] = int(
|
||||||
|
datetime.now().timestamp()
|
||||||
|
)
|
||||||
|
|
||||||
|
for i, item in enumerate(playlist):
|
||||||
|
item["idx"] = i
|
||||||
|
|
||||||
|
self.set_playlist_thumbnail()
|
||||||
|
self.upload_to_es()
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
def del_video(self, video_id):
|
||||||
|
playlist = self.json_data["playlist_entries"]
|
||||||
|
|
||||||
|
i = 0
|
||||||
|
while i < len(playlist):
|
||||||
|
if video_id == playlist[i]["youtube_id"]:
|
||||||
|
playlist.pop(i)
|
||||||
|
self.set_playlist_thumbnail()
|
||||||
|
i -= 1
|
||||||
|
i += 1
|
||||||
|
|
||||||
|
def get_video_index(self, video_id):
|
||||||
|
for i, child in enumerate(self.json_data["playlist_entries"]):
|
||||||
|
if child["youtube_id"] == video_id:
|
||||||
|
return i
|
||||||
|
return -1
|
||||||
|
|
||||||
|
def playlist_entries_contains(self, video_id):
|
||||||
|
return (
|
||||||
|
len(
|
||||||
|
list(
|
||||||
|
filter(
|
||||||
|
lambda x: x["youtube_id"] == video_id,
|
||||||
|
self.json_data["playlist_entries"],
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
> 0
|
||||||
|
)
|
||||||
|
|
||||||
|
def get_video_is_watched(self, video_id):
|
||||||
|
video = YoutubeVideo(video_id)
|
||||||
|
video.get_from_es()
|
||||||
|
return video.json_data["player"]["watched"]
|
||||||
|
|
||||||
|
def set_playlist_thumbnail(self):
|
||||||
|
playlist = self.json_data["playlist_entries"]
|
||||||
|
self.json_data["playlist_thumbnail"] = False
|
||||||
|
|
||||||
|
for video in playlist:
|
||||||
|
url = ThumbManager(video["youtube_id"]).vid_thumb_path()
|
||||||
|
if url is not None:
|
||||||
|
self.json_data["playlist_thumbnail"] = url
|
||||||
|
break
|
||||||
|
self.get_playlist_art()
|
||||||
|
|
||||||
|
def get_video_metadata(self, video_id):
|
||||||
|
video = YoutubeVideo(video_id)
|
||||||
|
video.get_from_es()
|
||||||
|
video_json_data = {
|
||||||
|
"youtube_id": video.json_data["youtube_id"],
|
||||||
|
"title": video.json_data["title"],
|
||||||
|
"uploader": video.json_data["channel"]["channel_name"],
|
||||||
|
"idx": 0,
|
||||||
|
"downloaded": "date_downloaded" in video.json_data
|
||||||
|
and video.json_data["date_downloaded"] > 0,
|
||||||
|
}
|
||||||
|
return video_json_data
|
||||||
|
|
|
@ -6,28 +6,37 @@ functionality:
|
||||||
|
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import shutil
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from time import sleep
|
from time import sleep
|
||||||
|
from typing import Callable, TypedDict
|
||||||
|
|
||||||
from home.src.download.queue import PendingList
|
from home.models import CustomPeriodicTask
|
||||||
from home.src.download.subscriptions import ChannelSubscription
|
from home.src.download.subscriptions import ChannelSubscription
|
||||||
from home.src.download.thumbnails import ThumbManager
|
from home.src.download.thumbnails import ThumbManager
|
||||||
from home.src.download.yt_dlp_base import CookieHandler
|
from home.src.download.yt_dlp_base import CookieHandler
|
||||||
from home.src.download.yt_dlp_handler import VideoDownloader
|
|
||||||
from home.src.es.connect import ElasticWrap, IndexPaginate
|
from home.src.es.connect import ElasticWrap, IndexPaginate
|
||||||
from home.src.index.channel import YoutubeChannel
|
from home.src.index.channel import YoutubeChannel
|
||||||
from home.src.index.comments import Comments
|
from home.src.index.comments import Comments
|
||||||
from home.src.index.playlist import YoutubePlaylist
|
from home.src.index.playlist import YoutubePlaylist
|
||||||
from home.src.index.video import YoutubeVideo
|
from home.src.index.video import YoutubeVideo
|
||||||
from home.src.ta.config import AppConfig
|
from home.src.ta.config import AppConfig
|
||||||
|
from home.src.ta.settings import EnvironmentSettings
|
||||||
from home.src.ta.ta_redis import RedisQueue
|
from home.src.ta.ta_redis import RedisQueue
|
||||||
|
|
||||||
|
|
||||||
|
class ReindexConfigType(TypedDict):
|
||||||
|
"""represents config type"""
|
||||||
|
|
||||||
|
index_name: str
|
||||||
|
queue_name: str
|
||||||
|
active_key: str
|
||||||
|
refresh_key: str
|
||||||
|
|
||||||
|
|
||||||
class ReindexBase:
|
class ReindexBase:
|
||||||
"""base config class for reindex task"""
|
"""base config class for reindex task"""
|
||||||
|
|
||||||
REINDEX_CONFIG = {
|
REINDEX_CONFIG: dict[str, ReindexConfigType] = {
|
||||||
"video": {
|
"video": {
|
||||||
"index_name": "ta_video",
|
"index_name": "ta_video",
|
||||||
"queue_name": "reindex:ta_video",
|
"queue_name": "reindex:ta_video",
|
||||||
|
@ -55,7 +64,7 @@ class ReindexBase:
|
||||||
self.config = AppConfig().config
|
self.config = AppConfig().config
|
||||||
self.now = int(datetime.now().timestamp())
|
self.now = int(datetime.now().timestamp())
|
||||||
|
|
||||||
def populate(self, all_ids, reindex_config):
|
def populate(self, all_ids, reindex_config: ReindexConfigType):
|
||||||
"""add all to reindex ids to redis queue"""
|
"""add all to reindex ids to redis queue"""
|
||||||
if not all_ids:
|
if not all_ids:
|
||||||
return
|
return
|
||||||
|
@ -66,11 +75,24 @@ class ReindexBase:
|
||||||
class ReindexPopulate(ReindexBase):
|
class ReindexPopulate(ReindexBase):
|
||||||
"""add outdated and recent documents to reindex queue"""
|
"""add outdated and recent documents to reindex queue"""
|
||||||
|
|
||||||
|
INTERVAL_DEFAIULT: int = 90
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.interval = self.config["scheduler"]["check_reindex_days"]
|
self.interval = self.INTERVAL_DEFAIULT
|
||||||
|
|
||||||
def add_recent(self):
|
def get_interval(self) -> None:
|
||||||
|
"""get reindex days interval from task"""
|
||||||
|
try:
|
||||||
|
task = CustomPeriodicTask.objects.get(name="check_reindex")
|
||||||
|
except CustomPeriodicTask.DoesNotExist:
|
||||||
|
return
|
||||||
|
|
||||||
|
task_config = task.task_config
|
||||||
|
if task_config.get("days"):
|
||||||
|
self.interval = task_config.get("days")
|
||||||
|
|
||||||
|
def add_recent(self) -> None:
|
||||||
"""add recent videos to refresh"""
|
"""add recent videos to refresh"""
|
||||||
gte = datetime.fromtimestamp(self.now - self.DAYS3).date().isoformat()
|
gte = datetime.fromtimestamp(self.now - self.DAYS3).date().isoformat()
|
||||||
must_list = [
|
must_list = [
|
||||||
|
@ -88,10 +110,10 @@ class ReindexPopulate(ReindexBase):
|
||||||
return
|
return
|
||||||
|
|
||||||
all_ids = [i["_source"]["youtube_id"] for i in hits]
|
all_ids = [i["_source"]["youtube_id"] for i in hits]
|
||||||
reindex_config = self.REINDEX_CONFIG.get("video")
|
reindex_config: ReindexConfigType = self.REINDEX_CONFIG["video"]
|
||||||
self.populate(all_ids, reindex_config)
|
self.populate(all_ids, reindex_config)
|
||||||
|
|
||||||
def add_outdated(self):
|
def add_outdated(self) -> None:
|
||||||
"""add outdated documents"""
|
"""add outdated documents"""
|
||||||
for reindex_config in self.REINDEX_CONFIG.values():
|
for reindex_config in self.REINDEX_CONFIG.values():
|
||||||
total_hits = self._get_total_hits(reindex_config)
|
total_hits = self._get_total_hits(reindex_config)
|
||||||
|
@ -100,17 +122,19 @@ class ReindexPopulate(ReindexBase):
|
||||||
self.populate(all_ids, reindex_config)
|
self.populate(all_ids, reindex_config)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _get_total_hits(reindex_config):
|
def _get_total_hits(reindex_config: ReindexConfigType) -> int:
|
||||||
"""get total hits from index"""
|
"""get total hits from index"""
|
||||||
index_name = reindex_config["index_name"]
|
index_name = reindex_config["index_name"]
|
||||||
active_key = reindex_config["active_key"]
|
active_key = reindex_config["active_key"]
|
||||||
path = f"{index_name}/_search?filter_path=hits.total"
|
data = {
|
||||||
data = {"query": {"match": {active_key: True}}}
|
"query": {"term": {active_key: {"value": True}}},
|
||||||
response, _ = ElasticWrap(path).post(data=data)
|
"_source": False,
|
||||||
total_hits = response["hits"]["total"]["value"]
|
}
|
||||||
return total_hits
|
total = IndexPaginate(index_name, data, keep_source=True).get_results()
|
||||||
|
|
||||||
def _get_daily_should(self, total_hits):
|
return len(total)
|
||||||
|
|
||||||
|
def _get_daily_should(self, total_hits: int) -> int:
|
||||||
"""calc how many should reindex daily"""
|
"""calc how many should reindex daily"""
|
||||||
daily_should = int((total_hits // self.interval + 1) * self.MULTIPLY)
|
daily_should = int((total_hits // self.interval + 1) * self.MULTIPLY)
|
||||||
if daily_should >= 10000:
|
if daily_should >= 10000:
|
||||||
|
@ -118,11 +142,13 @@ class ReindexPopulate(ReindexBase):
|
||||||
|
|
||||||
return daily_should
|
return daily_should
|
||||||
|
|
||||||
def _get_outdated_ids(self, reindex_config, daily_should):
|
def _get_outdated_ids(
|
||||||
|
self, reindex_config: ReindexConfigType, daily_should: int
|
||||||
|
) -> list[str]:
|
||||||
"""get outdated from index_name"""
|
"""get outdated from index_name"""
|
||||||
index_name = reindex_config["index_name"]
|
index_name = reindex_config["index_name"]
|
||||||
refresh_key = reindex_config["refresh_key"]
|
refresh_key = reindex_config["refresh_key"]
|
||||||
now_lte = self.now - self.interval * 24 * 60 * 60
|
now_lte = str(self.now - self.interval * 24 * 60 * 60)
|
||||||
must_list = [
|
must_list = [
|
||||||
{"match": {reindex_config["active_key"]: True}},
|
{"match": {reindex_config["active_key"]: True}},
|
||||||
{"range": {refresh_key: {"lte": now_lte}}},
|
{"range": {refresh_key: {"lte": now_lte}}},
|
||||||
|
@ -155,7 +181,7 @@ class ReindexManual(ReindexBase):
|
||||||
self.extract_videos = extract_videos
|
self.extract_videos = extract_videos
|
||||||
self.data = False
|
self.data = False
|
||||||
|
|
||||||
def extract_data(self, data):
|
def extract_data(self, data) -> None:
|
||||||
"""process data"""
|
"""process data"""
|
||||||
self.data = data
|
self.data = data
|
||||||
for key, values in self.data.items():
|
for key, values in self.data.items():
|
||||||
|
@ -166,7 +192,9 @@ class ReindexManual(ReindexBase):
|
||||||
|
|
||||||
self.process_index(reindex_config, values)
|
self.process_index(reindex_config, values)
|
||||||
|
|
||||||
def process_index(self, index_config, values):
|
def process_index(
|
||||||
|
self, index_config: ReindexConfigType, values: list[str]
|
||||||
|
) -> None:
|
||||||
"""process values per index"""
|
"""process values per index"""
|
||||||
index_name = index_config["index_name"]
|
index_name = index_config["index_name"]
|
||||||
if index_name == "ta_video":
|
if index_name == "ta_video":
|
||||||
|
@ -176,32 +204,35 @@ class ReindexManual(ReindexBase):
|
||||||
elif index_name == "ta_playlist":
|
elif index_name == "ta_playlist":
|
||||||
self._add_playlists(values)
|
self._add_playlists(values)
|
||||||
|
|
||||||
def _add_videos(self, values):
|
def _add_videos(self, values: list[str]) -> None:
|
||||||
"""add list of videos to reindex queue"""
|
"""add list of videos to reindex queue"""
|
||||||
if not values:
|
if not values:
|
||||||
return
|
return
|
||||||
|
|
||||||
RedisQueue("reindex:ta_video").add_list(values)
|
queue_name = self.REINDEX_CONFIG["video"]["queue_name"]
|
||||||
|
RedisQueue(queue_name).add_list(values)
|
||||||
|
|
||||||
def _add_channels(self, values):
|
def _add_channels(self, values: list[str]) -> None:
|
||||||
"""add list of channels to reindex queue"""
|
"""add list of channels to reindex queue"""
|
||||||
RedisQueue("reindex:ta_channel").add_list(values)
|
queue_name = self.REINDEX_CONFIG["channel"]["queue_name"]
|
||||||
|
RedisQueue(queue_name).add_list(values)
|
||||||
|
|
||||||
if self.extract_videos:
|
if self.extract_videos:
|
||||||
for channel_id in values:
|
for channel_id in values:
|
||||||
all_videos = self._get_channel_videos(channel_id)
|
all_videos = self._get_channel_videos(channel_id)
|
||||||
self._add_videos(all_videos)
|
self._add_videos(all_videos)
|
||||||
|
|
||||||
def _add_playlists(self, values):
|
def _add_playlists(self, values: list[str]) -> None:
|
||||||
"""add list of playlists to reindex queue"""
|
"""add list of playlists to reindex queue"""
|
||||||
RedisQueue("reindex:ta_playlist").add_list(values)
|
queue_name = self.REINDEX_CONFIG["playlist"]["queue_name"]
|
||||||
|
RedisQueue(queue_name).add_list(values)
|
||||||
|
|
||||||
if self.extract_videos:
|
if self.extract_videos:
|
||||||
for playlist_id in values:
|
for playlist_id in values:
|
||||||
all_videos = self._get_playlist_videos(playlist_id)
|
all_videos = self._get_playlist_videos(playlist_id)
|
||||||
self._add_videos(all_videos)
|
self._add_videos(all_videos)
|
||||||
|
|
||||||
def _get_channel_videos(self, channel_id):
|
def _get_channel_videos(self, channel_id: str) -> list[str]:
|
||||||
"""get all videos from channel"""
|
"""get all videos from channel"""
|
||||||
data = {
|
data = {
|
||||||
"query": {"term": {"channel.channel_id": {"value": channel_id}}},
|
"query": {"term": {"channel.channel_id": {"value": channel_id}}},
|
||||||
|
@ -210,7 +241,7 @@ class ReindexManual(ReindexBase):
|
||||||
all_results = IndexPaginate("ta_video", data).get_results()
|
all_results = IndexPaginate("ta_video", data).get_results()
|
||||||
return [i["youtube_id"] for i in all_results]
|
return [i["youtube_id"] for i in all_results]
|
||||||
|
|
||||||
def _get_playlist_videos(self, playlist_id):
|
def _get_playlist_videos(self, playlist_id: str) -> list[str]:
|
||||||
"""get all videos from playlist"""
|
"""get all videos from playlist"""
|
||||||
data = {
|
data = {
|
||||||
"query": {"term": {"playlist.keyword": {"value": playlist_id}}},
|
"query": {"term": {"playlist.keyword": {"value": playlist_id}}},
|
||||||
|
@ -226,37 +257,42 @@ class Reindex(ReindexBase):
|
||||||
def __init__(self, task=False):
|
def __init__(self, task=False):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.task = task
|
self.task = task
|
||||||
self.all_indexed_ids = False
|
self.processed = {
|
||||||
|
"videos": 0,
|
||||||
|
"channels": 0,
|
||||||
|
"playlists": 0,
|
||||||
|
}
|
||||||
|
|
||||||
def reindex_all(self):
|
def reindex_all(self) -> None:
|
||||||
"""reindex all in queue"""
|
"""reindex all in queue"""
|
||||||
if not self.cookie_is_valid():
|
if not self.cookie_is_valid():
|
||||||
print("[reindex] cookie invalid, exiting...")
|
print("[reindex] cookie invalid, exiting...")
|
||||||
return
|
return
|
||||||
|
|
||||||
for name, index_config in self.REINDEX_CONFIG.items():
|
for name, index_config in self.REINDEX_CONFIG.items():
|
||||||
if not RedisQueue(index_config["queue_name"]).has_item():
|
if not RedisQueue(index_config["queue_name"]).length():
|
||||||
continue
|
continue
|
||||||
|
|
||||||
total = RedisQueue(index_config["queue_name"]).length()
|
self.reindex_type(name, index_config)
|
||||||
while True:
|
|
||||||
has_next = self.reindex_index(name, index_config, total)
|
|
||||||
if not has_next:
|
|
||||||
break
|
|
||||||
|
|
||||||
def reindex_index(self, name, index_config, total):
|
def reindex_type(self, name: str, index_config: ReindexConfigType) -> None:
|
||||||
"""reindex all of a single index"""
|
"""reindex all of a single index"""
|
||||||
reindex = self.get_reindex_map(index_config["index_name"])
|
reindex = self._get_reindex_map(index_config["index_name"])
|
||||||
youtube_id = RedisQueue(index_config["queue_name"]).get_next()
|
queue = RedisQueue(index_config["queue_name"])
|
||||||
if youtube_id:
|
while True:
|
||||||
self._notify(name, index_config, total)
|
total = queue.max_score()
|
||||||
|
youtube_id, idx = queue.get_next()
|
||||||
|
if not youtube_id or not idx or not total:
|
||||||
|
break
|
||||||
|
|
||||||
|
if self.task:
|
||||||
|
self._notify(name, total, idx)
|
||||||
|
|
||||||
reindex(youtube_id)
|
reindex(youtube_id)
|
||||||
sleep_interval = self.config["downloads"].get("sleep_interval", 0)
|
sleep_interval = self.config["downloads"].get("sleep_interval", 0)
|
||||||
sleep(sleep_interval)
|
sleep(sleep_interval)
|
||||||
|
|
||||||
return bool(youtube_id)
|
def _get_reindex_map(self, index_name: str) -> Callable:
|
||||||
|
|
||||||
def get_reindex_map(self, index_name):
|
|
||||||
"""return def to run for index"""
|
"""return def to run for index"""
|
||||||
def_map = {
|
def_map = {
|
||||||
"ta_video": self._reindex_single_video,
|
"ta_video": self._reindex_single_video,
|
||||||
|
@ -264,34 +300,30 @@ class Reindex(ReindexBase):
|
||||||
"ta_playlist": self._reindex_single_playlist,
|
"ta_playlist": self._reindex_single_playlist,
|
||||||
}
|
}
|
||||||
|
|
||||||
return def_map.get(index_name)
|
return def_map[index_name]
|
||||||
|
|
||||||
def _notify(self, name, index_config, total):
|
def _notify(self, name: str, total: int, idx: int) -> None:
|
||||||
"""send notification back to task"""
|
"""send notification back to task"""
|
||||||
remaining = RedisQueue(index_config["queue_name"]).length()
|
|
||||||
idx = total - remaining
|
|
||||||
message = [f"Reindexing {name.title()}s {idx}/{total}"]
|
message = [f"Reindexing {name.title()}s {idx}/{total}"]
|
||||||
progress = idx / total
|
progress = idx / total
|
||||||
self.task.send_progress(message, progress=progress)
|
self.task.send_progress(message, progress=progress)
|
||||||
|
|
||||||
def _reindex_single_video(self, youtube_id):
|
def _reindex_single_video(self, youtube_id: str) -> None:
|
||||||
"""wrapper to handle channel name changes"""
|
|
||||||
try:
|
|
||||||
self._reindex_single_video_call(youtube_id)
|
|
||||||
except FileNotFoundError:
|
|
||||||
ChannelUrlFixer(youtube_id, self.config).run()
|
|
||||||
self._reindex_single_video_call(youtube_id)
|
|
||||||
|
|
||||||
def _reindex_single_video_call(self, youtube_id):
|
|
||||||
"""refresh data for single video"""
|
"""refresh data for single video"""
|
||||||
video = YoutubeVideo(youtube_id)
|
video = YoutubeVideo(youtube_id)
|
||||||
|
|
||||||
# read current state
|
# read current state
|
||||||
video.get_from_es()
|
video.get_from_es()
|
||||||
|
if not video.json_data:
|
||||||
|
return
|
||||||
|
|
||||||
es_meta = video.json_data.copy()
|
es_meta = video.json_data.copy()
|
||||||
|
|
||||||
# get new
|
# get new
|
||||||
video.build_json()
|
media_url = os.path.join(
|
||||||
|
EnvironmentSettings.MEDIA_DIR, es_meta["media_url"]
|
||||||
|
)
|
||||||
|
video.build_json(media_path=media_url)
|
||||||
if not video.youtube_meta:
|
if not video.youtube_meta:
|
||||||
video.deactivate()
|
video.deactivate()
|
||||||
return
|
return
|
||||||
|
@ -307,76 +339,63 @@ class Reindex(ReindexBase):
|
||||||
video.json_data["playlist"] = es_meta.get("playlist")
|
video.json_data["playlist"] = es_meta.get("playlist")
|
||||||
|
|
||||||
video.upload_to_es()
|
video.upload_to_es()
|
||||||
if es_meta.get("media_url") != video.json_data["media_url"]:
|
|
||||||
self._rename_media_file(
|
|
||||||
es_meta.get("media_url"), video.json_data["media_url"]
|
|
||||||
)
|
|
||||||
|
|
||||||
thumb_handler = ThumbManager(youtube_id)
|
thumb_handler = ThumbManager(youtube_id)
|
||||||
thumb_handler.delete_video_thumb()
|
thumb_handler.delete_video_thumb()
|
||||||
thumb_handler.download_video_thumb(video.json_data["vid_thumb_url"])
|
thumb_handler.download_video_thumb(video.json_data["vid_thumb_url"])
|
||||||
|
|
||||||
Comments(youtube_id, config=self.config).reindex_comments()
|
Comments(youtube_id, config=self.config).reindex_comments()
|
||||||
|
self.processed["videos"] += 1
|
||||||
|
|
||||||
return
|
def _reindex_single_channel(self, channel_id: str) -> None:
|
||||||
|
|
||||||
def _rename_media_file(self, media_url_is, media_url_should):
|
|
||||||
"""handle title change"""
|
|
||||||
print(f"[reindex] fix media_url {media_url_is} to {media_url_should}")
|
|
||||||
videos = self.config["application"]["videos"]
|
|
||||||
old_path = os.path.join(videos, media_url_is)
|
|
||||||
new_path = os.path.join(videos, media_url_should)
|
|
||||||
os.rename(old_path, new_path)
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _reindex_single_channel(channel_id):
|
|
||||||
"""refresh channel data and sync to videos"""
|
"""refresh channel data and sync to videos"""
|
||||||
|
# read current state
|
||||||
channel = YoutubeChannel(channel_id)
|
channel = YoutubeChannel(channel_id)
|
||||||
channel.get_from_es()
|
channel.get_from_es()
|
||||||
subscribed = channel.json_data["channel_subscribed"]
|
|
||||||
overwrites = channel.json_data.get("channel_overwrites", False)
|
|
||||||
channel.get_from_youtube()
|
|
||||||
if not channel.json_data:
|
if not channel.json_data:
|
||||||
|
return
|
||||||
|
|
||||||
|
es_meta = channel.json_data.copy()
|
||||||
|
|
||||||
|
# get new
|
||||||
|
channel.get_from_youtube()
|
||||||
|
if not channel.youtube_meta:
|
||||||
channel.deactivate()
|
channel.deactivate()
|
||||||
channel.get_from_es()
|
channel.get_from_es()
|
||||||
channel.sync_to_videos()
|
channel.sync_to_videos()
|
||||||
return
|
return
|
||||||
|
|
||||||
channel.json_data["channel_subscribed"] = subscribed
|
channel.process_youtube_meta()
|
||||||
|
channel.get_channel_art()
|
||||||
|
|
||||||
|
# add back
|
||||||
|
channel.json_data["channel_subscribed"] = es_meta["channel_subscribed"]
|
||||||
|
overwrites = es_meta.get("channel_overwrites")
|
||||||
if overwrites:
|
if overwrites:
|
||||||
channel.json_data["channel_overwrites"] = overwrites
|
channel.json_data["channel_overwrites"] = overwrites
|
||||||
|
|
||||||
channel.upload_to_es()
|
channel.upload_to_es()
|
||||||
channel.sync_to_videos()
|
|
||||||
|
|
||||||
ChannelFullScan(channel_id).scan()
|
ChannelFullScan(channel_id).scan()
|
||||||
|
self.processed["channels"] += 1
|
||||||
|
|
||||||
def _reindex_single_playlist(self, playlist_id):
|
def _reindex_single_playlist(self, playlist_id: str) -> None:
|
||||||
"""refresh playlist data"""
|
"""refresh playlist data"""
|
||||||
self._get_all_videos()
|
|
||||||
playlist = YoutubePlaylist(playlist_id)
|
playlist = YoutubePlaylist(playlist_id)
|
||||||
playlist.get_from_es()
|
playlist.get_from_es()
|
||||||
subscribed = playlist.json_data["playlist_subscribed"]
|
if (
|
||||||
playlist.all_youtube_ids = self.all_indexed_ids
|
not playlist.json_data
|
||||||
playlist.build_json(scrape=True)
|
or playlist.json_data["playlist_type"] == "custom"
|
||||||
if not playlist.json_data:
|
):
|
||||||
|
return
|
||||||
|
|
||||||
|
is_active = playlist.update_playlist()
|
||||||
|
if not is_active:
|
||||||
playlist.deactivate()
|
playlist.deactivate()
|
||||||
return
|
return
|
||||||
|
|
||||||
playlist.json_data["playlist_subscribed"] = subscribed
|
self.processed["playlists"] += 1
|
||||||
playlist.upload_to_es()
|
|
||||||
return
|
|
||||||
|
|
||||||
def _get_all_videos(self):
|
def cookie_is_valid(self) -> bool:
|
||||||
"""add all videos for playlist index validation"""
|
|
||||||
if self.all_indexed_ids:
|
|
||||||
return
|
|
||||||
|
|
||||||
handler = PendingList()
|
|
||||||
handler.get_download()
|
|
||||||
handler.get_indexed()
|
|
||||||
self.all_indexed_ids = [i["youtube_id"] for i in handler.all_videos]
|
|
||||||
|
|
||||||
def cookie_is_valid(self):
|
|
||||||
"""return true if cookie is enabled and valid"""
|
"""return true if cookie is enabled and valid"""
|
||||||
if not self.config["downloads"]["cookie_import"]:
|
if not self.config["downloads"]["cookie_import"]:
|
||||||
# is not activated, continue reindex
|
# is not activated, continue reindex
|
||||||
|
@ -385,6 +404,18 @@ class Reindex(ReindexBase):
|
||||||
valid = CookieHandler(self.config).validate()
|
valid = CookieHandler(self.config).validate()
|
||||||
return valid
|
return valid
|
||||||
|
|
||||||
|
def build_message(self) -> str:
|
||||||
|
"""build progress message"""
|
||||||
|
message = ""
|
||||||
|
for key, value in self.processed.items():
|
||||||
|
if value:
|
||||||
|
message = message + f"{value} {key}, "
|
||||||
|
|
||||||
|
if message:
|
||||||
|
message = f"reindexed {message.rstrip(', ')}"
|
||||||
|
|
||||||
|
return message
|
||||||
|
|
||||||
|
|
||||||
class ReindexProgress(ReindexBase):
|
class ReindexProgress(ReindexBase):
|
||||||
"""
|
"""
|
||||||
|
@ -403,7 +434,7 @@ class ReindexProgress(ReindexBase):
|
||||||
self.request_type = request_type
|
self.request_type = request_type
|
||||||
self.request_id = request_id
|
self.request_id = request_id
|
||||||
|
|
||||||
def get_progress(self):
|
def get_progress(self) -> dict:
|
||||||
"""get progress from task"""
|
"""get progress from task"""
|
||||||
queue_name, request_type = self._get_queue_name()
|
queue_name, request_type = self._get_queue_name()
|
||||||
total = self._get_total_in_queue(queue_name)
|
total = self._get_total_in_queue(queue_name)
|
||||||
|
@ -460,65 +491,6 @@ class ReindexProgress(ReindexBase):
|
||||||
return state_dict
|
return state_dict
|
||||||
|
|
||||||
|
|
||||||
class ChannelUrlFixer:
|
|
||||||
"""fix not matching channel names in reindex"""
|
|
||||||
|
|
||||||
def __init__(self, youtube_id, config):
|
|
||||||
self.youtube_id = youtube_id
|
|
||||||
self.config = config
|
|
||||||
self.video = False
|
|
||||||
|
|
||||||
def run(self):
|
|
||||||
"""check and run if needed"""
|
|
||||||
print(f"{self.youtube_id}: failed to build channel path, try to fix.")
|
|
||||||
video_path_is, video_folder_is = self.get_as_is()
|
|
||||||
if not os.path.exists(video_path_is):
|
|
||||||
print(f"giving up reindex, video in video: {self.video.json_data}")
|
|
||||||
raise ValueError
|
|
||||||
|
|
||||||
_, video_folder_should = self.get_as_should()
|
|
||||||
|
|
||||||
if video_folder_is != video_folder_should:
|
|
||||||
self.process(video_path_is)
|
|
||||||
else:
|
|
||||||
print(f"{self.youtube_id}: skip channel url fixer")
|
|
||||||
|
|
||||||
def get_as_is(self):
|
|
||||||
"""get video object as is"""
|
|
||||||
self.video = YoutubeVideo(self.youtube_id)
|
|
||||||
self.video.get_from_es()
|
|
||||||
video_path_is = os.path.join(
|
|
||||||
self.config["application"]["videos"],
|
|
||||||
self.video.json_data["media_url"],
|
|
||||||
)
|
|
||||||
video_folder_is = os.path.split(video_path_is)[0]
|
|
||||||
|
|
||||||
return video_path_is, video_folder_is
|
|
||||||
|
|
||||||
def get_as_should(self):
|
|
||||||
"""add fresh metadata from remote"""
|
|
||||||
self.video.get_from_youtube()
|
|
||||||
self.video.add_file_path()
|
|
||||||
|
|
||||||
video_path_should = os.path.join(
|
|
||||||
self.config["application"]["videos"],
|
|
||||||
self.video.json_data["media_url"],
|
|
||||||
)
|
|
||||||
video_folder_should = os.path.split(video_path_should)[0]
|
|
||||||
return video_path_should, video_folder_should
|
|
||||||
|
|
||||||
def process(self, video_path_is):
|
|
||||||
"""fix filepath"""
|
|
||||||
print(f"{self.youtube_id}: fixing channel rename.")
|
|
||||||
cache_dir = self.config["application"]["cache_dir"]
|
|
||||||
new_path = os.path.join(
|
|
||||||
cache_dir, "download", self.youtube_id + ".mp4"
|
|
||||||
)
|
|
||||||
shutil.move(video_path_is, new_path, copy_function=shutil.copyfile)
|
|
||||||
VideoDownloader().move_to_archive(self.video.json_data)
|
|
||||||
self.video.update_media_url()
|
|
||||||
|
|
||||||
|
|
||||||
class ChannelFullScan:
|
class ChannelFullScan:
|
||||||
"""
|
"""
|
||||||
update from v0.3.0 to v0.3.1
|
update from v0.3.0 to v0.3.1
|
||||||
|
|
|
@ -12,6 +12,7 @@ from datetime import datetime
|
||||||
import requests
|
import requests
|
||||||
from home.src.es.connect import ElasticWrap
|
from home.src.es.connect import ElasticWrap
|
||||||
from home.src.ta.helper import requests_headers
|
from home.src.ta.helper import requests_headers
|
||||||
|
from home.src.ta.settings import EnvironmentSettings
|
||||||
|
|
||||||
|
|
||||||
class YoutubeSubtitle:
|
class YoutubeSubtitle:
|
||||||
|
@ -62,7 +63,12 @@ class YoutubeSubtitle:
|
||||||
if not all_formats:
|
if not all_formats:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
subtitle = [i for i in all_formats if i["ext"] == "json3"][0]
|
subtitle_json3 = [i for i in all_formats if i["ext"] == "json3"]
|
||||||
|
if not subtitle_json3:
|
||||||
|
print(f"{self.video.youtube_id}-{lang}: json3 not processed")
|
||||||
|
return False
|
||||||
|
|
||||||
|
subtitle = subtitle_json3[0]
|
||||||
subtitle.update(
|
subtitle.update(
|
||||||
{"lang": lang, "source": "auto", "media_url": media_url}
|
{"lang": lang, "source": "auto", "media_url": media_url}
|
||||||
)
|
)
|
||||||
|
@ -108,7 +114,7 @@ class YoutubeSubtitle:
|
||||||
|
|
||||||
def download_subtitles(self, relevant_subtitles):
|
def download_subtitles(self, relevant_subtitles):
|
||||||
"""download subtitle files to archive"""
|
"""download subtitle files to archive"""
|
||||||
videos_base = self.video.config["application"]["videos"]
|
videos_base = EnvironmentSettings.MEDIA_DIR
|
||||||
indexed = []
|
indexed = []
|
||||||
for subtitle in relevant_subtitles:
|
for subtitle in relevant_subtitles:
|
||||||
dest_path = os.path.join(videos_base, subtitle["media_url"])
|
dest_path = os.path.join(videos_base, subtitle["media_url"])
|
||||||
|
@ -122,6 +128,10 @@ class YoutubeSubtitle:
|
||||||
print(response.text)
|
print(response.text)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
if not response.text:
|
||||||
|
print(f"{self.video.youtube_id}: skip empty subtitle")
|
||||||
|
continue
|
||||||
|
|
||||||
parser = SubtitleParser(response.text, lang, source)
|
parser = SubtitleParser(response.text, lang, source)
|
||||||
parser.process()
|
parser.process()
|
||||||
if not parser.all_cues:
|
if not parser.all_cues:
|
||||||
|
@ -144,8 +154,8 @@ class YoutubeSubtitle:
|
||||||
with open(dest_path, "w", encoding="utf-8") as subfile:
|
with open(dest_path, "w", encoding="utf-8") as subfile:
|
||||||
subfile.write(subtitle_str)
|
subfile.write(subtitle_str)
|
||||||
|
|
||||||
host_uid = self.video.config["application"]["HOST_UID"]
|
host_uid = EnvironmentSettings.HOST_UID
|
||||||
host_gid = self.video.config["application"]["HOST_GID"]
|
host_gid = EnvironmentSettings.HOST_GID
|
||||||
if host_uid and host_gid:
|
if host_uid and host_gid:
|
||||||
os.chown(dest_path, host_uid, host_gid)
|
os.chown(dest_path, host_uid, host_gid)
|
||||||
|
|
||||||
|
@ -157,7 +167,7 @@ class YoutubeSubtitle:
|
||||||
def delete(self, subtitles=False):
|
def delete(self, subtitles=False):
|
||||||
"""delete subtitles from index and filesystem"""
|
"""delete subtitles from index and filesystem"""
|
||||||
youtube_id = self.video.youtube_id
|
youtube_id = self.video.youtube_id
|
||||||
videos_base = self.video.config["application"]["videos"]
|
videos_base = EnvironmentSettings.MEDIA_DIR
|
||||||
# delete files
|
# delete files
|
||||||
if subtitles:
|
if subtitles:
|
||||||
files = [i["media_url"] for i in subtitles]
|
files = [i["media_url"] for i in subtitles]
|
||||||
|
|
|
@ -16,8 +16,10 @@ from home.src.index import playlist as ta_playlist
|
||||||
from home.src.index.generic import YouTubeItem
|
from home.src.index.generic import YouTubeItem
|
||||||
from home.src.index.subtitle import YoutubeSubtitle
|
from home.src.index.subtitle import YoutubeSubtitle
|
||||||
from home.src.index.video_constants import VideoTypeEnum
|
from home.src.index.video_constants import VideoTypeEnum
|
||||||
from home.src.ta.helper import DurationConverter, clean_string, randomizor
|
from home.src.index.video_streams import MediaStreamExtractor
|
||||||
from home.src.ta.ta_redis import RedisArchivist
|
from home.src.ta.helper import get_duration_sec, get_duration_str, randomizor
|
||||||
|
from home.src.ta.settings import EnvironmentSettings
|
||||||
|
from home.src.ta.users import UserConfig
|
||||||
from ryd_client import ryd_client
|
from ryd_client import ryd_client
|
||||||
|
|
||||||
|
|
||||||
|
@ -31,17 +33,16 @@ class SponsorBlock:
|
||||||
self.user_agent = f"{settings.TA_UPSTREAM} {settings.TA_VERSION}"
|
self.user_agent = f"{settings.TA_UPSTREAM} {settings.TA_VERSION}"
|
||||||
self.last_refresh = int(datetime.now().timestamp())
|
self.last_refresh = int(datetime.now().timestamp())
|
||||||
|
|
||||||
def get_sb_id(self):
|
def get_sb_id(self) -> str:
|
||||||
"""get sponsorblock userid or generate if needed"""
|
"""get sponsorblock for the userid or generate if needed"""
|
||||||
if not self.user_id:
|
if not self.user_id:
|
||||||
print("missing request user id")
|
raise ValueError("missing request user id")
|
||||||
raise ValueError
|
|
||||||
|
|
||||||
key = f"{self.user_id}:id_sponsorblock"
|
user = UserConfig(self.user_id)
|
||||||
sb_id = RedisArchivist().get_message(key)
|
sb_id = user.get_value("sponsorblock_id")
|
||||||
if not sb_id["status"]:
|
if not sb_id:
|
||||||
sb_id = {"status": randomizor(32)}
|
sb_id = randomizor(32)
|
||||||
RedisArchivist().set_message(key, sb_id)
|
user.set_value("sponsorblock_id", sb_id)
|
||||||
|
|
||||||
return sb_id
|
return sb_id
|
||||||
|
|
||||||
|
@ -87,7 +88,7 @@ class SponsorBlock:
|
||||||
|
|
||||||
def post_timestamps(self, youtube_id, start_time, end_time):
|
def post_timestamps(self, youtube_id, start_time, end_time):
|
||||||
"""post timestamps to api"""
|
"""post timestamps to api"""
|
||||||
user_id = self.get_sb_id().get("status")
|
user_id = self.get_sb_id()
|
||||||
data = {
|
data = {
|
||||||
"videoID": youtube_id,
|
"videoID": youtube_id,
|
||||||
"startTime": start_time,
|
"startTime": start_time,
|
||||||
|
@ -104,7 +105,7 @@ class SponsorBlock:
|
||||||
|
|
||||||
def vote_on_segment(self, uuid, vote):
|
def vote_on_segment(self, uuid, vote):
|
||||||
"""send vote on existing segment"""
|
"""send vote on existing segment"""
|
||||||
user_id = self.get_sb_id().get("status")
|
user_id = self.get_sb_id()
|
||||||
data = {
|
data = {
|
||||||
"UUID": uuid,
|
"UUID": uuid,
|
||||||
"userID": user_id,
|
"userID": user_id,
|
||||||
|
@ -124,17 +125,10 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
|
||||||
index_name = "ta_video"
|
index_name = "ta_video"
|
||||||
yt_base = "https://www.youtube.com/watch?v="
|
yt_base = "https://www.youtube.com/watch?v="
|
||||||
|
|
||||||
def __init__(
|
def __init__(self, youtube_id, video_type=VideoTypeEnum.VIDEOS):
|
||||||
self,
|
|
||||||
youtube_id,
|
|
||||||
video_overwrites=False,
|
|
||||||
video_type=VideoTypeEnum.VIDEOS,
|
|
||||||
):
|
|
||||||
super().__init__(youtube_id)
|
super().__init__(youtube_id)
|
||||||
self.channel_id = False
|
self.channel_id = False
|
||||||
self.video_overwrites = video_overwrites
|
|
||||||
self.video_type = video_type
|
self.video_type = video_type
|
||||||
self.es_path = f"{self.index_name}/_doc/{youtube_id}"
|
|
||||||
self.offline_import = False
|
self.offline_import = False
|
||||||
|
|
||||||
def build_json(self, youtube_meta_overwrite=False, media_path=False):
|
def build_json(self, youtube_meta_overwrite=False, media_path=False):
|
||||||
|
@ -147,11 +141,12 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
|
||||||
self.youtube_meta = youtube_meta_overwrite
|
self.youtube_meta = youtube_meta_overwrite
|
||||||
self.offline_import = True
|
self.offline_import = True
|
||||||
|
|
||||||
self._process_youtube_meta()
|
self.process_youtube_meta()
|
||||||
self._add_channel()
|
self._add_channel()
|
||||||
self._add_stats()
|
self._add_stats()
|
||||||
self.add_file_path()
|
self.add_file_path()
|
||||||
self.add_player(media_path)
|
self.add_player(media_path)
|
||||||
|
self.add_streams(media_path)
|
||||||
if self.config["downloads"]["integrate_ryd"]:
|
if self.config["downloads"]["integrate_ryd"]:
|
||||||
self._get_ryd_stats()
|
self._get_ryd_stats()
|
||||||
|
|
||||||
|
@ -164,18 +159,18 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
|
||||||
"""check if need to run sponsor block"""
|
"""check if need to run sponsor block"""
|
||||||
integrate = self.config["downloads"]["integrate_sponsorblock"]
|
integrate = self.config["downloads"]["integrate_sponsorblock"]
|
||||||
|
|
||||||
if self.video_overwrites:
|
if overwrite := self.json_data["channel"].get("channel_overwrites"):
|
||||||
single_overwrite = self.video_overwrites.get(self.youtube_id)
|
if not overwrite:
|
||||||
if not single_overwrite:
|
|
||||||
return integrate
|
return integrate
|
||||||
|
|
||||||
if "integrate_sponsorblock" in single_overwrite:
|
if "integrate_sponsorblock" in overwrite:
|
||||||
return single_overwrite.get("integrate_sponsorblock")
|
return overwrite.get("integrate_sponsorblock")
|
||||||
|
|
||||||
return integrate
|
return integrate
|
||||||
|
|
||||||
def _process_youtube_meta(self):
|
def process_youtube_meta(self):
|
||||||
"""extract relevant fields from youtube"""
|
"""extract relevant fields from youtube"""
|
||||||
|
self._validate_id()
|
||||||
# extract
|
# extract
|
||||||
self.channel_id = self.youtube_meta["channel_id"]
|
self.channel_id = self.youtube_meta["channel_id"]
|
||||||
upload_date = self.youtube_meta["upload_date"]
|
upload_date = self.youtube_meta["upload_date"]
|
||||||
|
@ -187,11 +182,11 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
|
||||||
# build json_data basics
|
# build json_data basics
|
||||||
self.json_data = {
|
self.json_data = {
|
||||||
"title": self.youtube_meta["title"],
|
"title": self.youtube_meta["title"],
|
||||||
"description": self.youtube_meta["description"],
|
"description": self.youtube_meta.get("description", ""),
|
||||||
"category": self.youtube_meta["categories"],
|
"category": self.youtube_meta.get("categories", []),
|
||||||
"vid_thumb_url": self.youtube_meta["thumbnail"],
|
"vid_thumb_url": self.youtube_meta["thumbnail"],
|
||||||
"vid_thumb_base64": base64_blur,
|
"vid_thumb_base64": base64_blur,
|
||||||
"tags": self.youtube_meta["tags"],
|
"tags": self.youtube_meta.get("tags", []),
|
||||||
"published": published,
|
"published": published,
|
||||||
"vid_last_refresh": last_refresh,
|
"vid_last_refresh": last_refresh,
|
||||||
"date_downloaded": last_refresh,
|
"date_downloaded": last_refresh,
|
||||||
|
@ -201,6 +196,19 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
|
||||||
"active": True,
|
"active": True,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def _validate_id(self):
|
||||||
|
"""validate expected video ID, raise value error on mismatch"""
|
||||||
|
remote_id = self.youtube_meta["id"]
|
||||||
|
|
||||||
|
if not self.youtube_id == remote_id:
|
||||||
|
# unexpected redirect
|
||||||
|
message = (
|
||||||
|
f"[reindex][{self.youtube_id}] got an unexpected redirect "
|
||||||
|
+ f"to {remote_id}, you are probably getting blocked by YT. "
|
||||||
|
"See FAQ for more details."
|
||||||
|
)
|
||||||
|
raise ValueError(message)
|
||||||
|
|
||||||
def _add_channel(self):
|
def _add_channel(self):
|
||||||
"""add channel dict to video json_data"""
|
"""add channel dict to video json_data"""
|
||||||
channel = ta_channel.YoutubeChannel(self.channel_id)
|
channel = ta_channel.YoutubeChannel(self.channel_id)
|
||||||
|
@ -209,87 +217,64 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
|
||||||
|
|
||||||
def _add_stats(self):
|
def _add_stats(self):
|
||||||
"""add stats dicst to json_data"""
|
"""add stats dicst to json_data"""
|
||||||
# likes
|
stats = {
|
||||||
like_count = self.youtube_meta.get("like_count", 0)
|
"view_count": self.youtube_meta.get("view_count", 0),
|
||||||
dislike_count = self.youtube_meta.get("dislike_count", 0)
|
"like_count": self.youtube_meta.get("like_count", 0),
|
||||||
average_rating = self.youtube_meta.get("average_rating", 0)
|
"dislike_count": self.youtube_meta.get("dislike_count", 0),
|
||||||
self.json_data.update(
|
"average_rating": self.youtube_meta.get("average_rating", 0),
|
||||||
{
|
}
|
||||||
"stats": {
|
self.json_data.update({"stats": stats})
|
||||||
"view_count": self.youtube_meta["view_count"],
|
|
||||||
"like_count": like_count,
|
|
||||||
"dislike_count": dislike_count,
|
|
||||||
"average_rating": average_rating,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
def build_dl_cache_path(self):
|
def build_dl_cache_path(self):
|
||||||
"""find video path in dl cache"""
|
"""find video path in dl cache"""
|
||||||
cache_dir = self.app_conf["cache_dir"]
|
cache_dir = EnvironmentSettings.CACHE_DIR
|
||||||
cache_path = f"{cache_dir}/download/"
|
video_id = self.json_data["youtube_id"]
|
||||||
all_cached = os.listdir(cache_path)
|
cache_path = f"{cache_dir}/download/{video_id}.mp4"
|
||||||
for file_cached in all_cached:
|
if os.path.exists(cache_path):
|
||||||
if self.youtube_id in file_cached:
|
return cache_path
|
||||||
vid_path = os.path.join(cache_path, file_cached)
|
|
||||||
return vid_path
|
channel_path = os.path.join(
|
||||||
|
EnvironmentSettings.MEDIA_DIR,
|
||||||
|
self.json_data["channel"]["channel_id"],
|
||||||
|
f"{video_id}.mp4",
|
||||||
|
)
|
||||||
|
if os.path.exists(channel_path):
|
||||||
|
return channel_path
|
||||||
|
|
||||||
raise FileNotFoundError
|
raise FileNotFoundError
|
||||||
|
|
||||||
def add_player(self, media_path=False):
|
def add_player(self, media_path=False):
|
||||||
"""add player information for new videos"""
|
"""add player information for new videos"""
|
||||||
vid_path = self._get_vid_path(media_path)
|
vid_path = media_path or self.build_dl_cache_path()
|
||||||
|
duration = get_duration_sec(vid_path)
|
||||||
|
|
||||||
duration_handler = DurationConverter()
|
|
||||||
duration = duration_handler.get_sec(vid_path)
|
|
||||||
duration_str = duration_handler.get_str(duration)
|
|
||||||
self.json_data.update(
|
self.json_data.update(
|
||||||
{
|
{
|
||||||
"player": {
|
"player": {
|
||||||
"watched": False,
|
"watched": False,
|
||||||
"duration": duration,
|
"duration": duration,
|
||||||
"duration_str": duration_str,
|
"duration_str": get_duration_str(duration),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
def _get_vid_path(self, media_path=False):
|
def add_streams(self, media_path=False):
|
||||||
"""get path of media file"""
|
"""add stream metadata"""
|
||||||
if media_path:
|
vid_path = media_path or self.build_dl_cache_path()
|
||||||
return media_path
|
media = MediaStreamExtractor(vid_path)
|
||||||
|
self.json_data.update(
|
||||||
try:
|
{
|
||||||
# when indexing from download task
|
"streams": media.extract_metadata(),
|
||||||
vid_path = self.build_dl_cache_path()
|
"media_size": media.get_file_size(),
|
||||||
except FileNotFoundError as err:
|
}
|
||||||
# when reindexing needs to handle title rename
|
)
|
||||||
channel = os.path.split(self.json_data["media_url"])[0]
|
|
||||||
channel_dir = os.path.join(self.app_conf["videos"], channel)
|
|
||||||
all_files = os.listdir(channel_dir)
|
|
||||||
for file in all_files:
|
|
||||||
if self.youtube_id in file and file.endswith(".mp4"):
|
|
||||||
vid_path = os.path.join(channel_dir, file)
|
|
||||||
break
|
|
||||||
else:
|
|
||||||
raise FileNotFoundError("could not find video file") from err
|
|
||||||
|
|
||||||
return vid_path
|
|
||||||
|
|
||||||
def add_file_path(self):
|
def add_file_path(self):
|
||||||
"""build media_url for where file will be located"""
|
"""build media_url for where file will be located"""
|
||||||
channel_name = self.json_data["channel"]["channel_name"]
|
self.json_data["media_url"] = os.path.join(
|
||||||
clean_channel_name = clean_string(channel_name)
|
self.json_data["channel"]["channel_id"],
|
||||||
if len(clean_channel_name) <= 3:
|
self.json_data["youtube_id"] + ".mp4",
|
||||||
# fall back to channel id
|
)
|
||||||
clean_channel_name = self.json_data["channel"]["channel_id"]
|
|
||||||
|
|
||||||
timestamp = self.json_data["published"].replace("-", "")
|
|
||||||
youtube_id = self.json_data["youtube_id"]
|
|
||||||
title = self.json_data["title"]
|
|
||||||
clean_title = clean_string(title)
|
|
||||||
filename = f"{timestamp}_{youtube_id}_{clean_title}.mp4"
|
|
||||||
media_url = os.path.join(clean_channel_name, filename)
|
|
||||||
self.json_data["media_url"] = media_url
|
|
||||||
|
|
||||||
def delete_media_file(self):
|
def delete_media_file(self):
|
||||||
"""delete video file, meta data"""
|
"""delete video file, meta data"""
|
||||||
|
@ -298,7 +283,7 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
|
||||||
if not self.json_data:
|
if not self.json_data:
|
||||||
raise FileNotFoundError
|
raise FileNotFoundError
|
||||||
|
|
||||||
video_base = self.app_conf["videos"]
|
video_base = EnvironmentSettings.MEDIA_DIR
|
||||||
media_url = self.json_data.get("media_url")
|
media_url = self.json_data.get("media_url")
|
||||||
file_path = os.path.join(video_base, media_url)
|
file_path = os.path.join(video_base, media_url)
|
||||||
try:
|
try:
|
||||||
|
@ -327,6 +312,8 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
|
||||||
playlist.json_data["playlist_entries"][idx].update(
|
playlist.json_data["playlist_entries"][idx].update(
|
||||||
{"downloaded": False}
|
{"downloaded": False}
|
||||||
)
|
)
|
||||||
|
if playlist.json_data["playlist_type"] == "custom":
|
||||||
|
playlist.del_video(self.youtube_id)
|
||||||
playlist.upload_to_es()
|
playlist.upload_to_es()
|
||||||
|
|
||||||
def delete_subtitles(self, subtitles=False):
|
def delete_subtitles(self, subtitles=False):
|
||||||
|
@ -405,13 +392,9 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
|
||||||
_, _ = ElasticWrap(path).post(data=data)
|
_, _ = ElasticWrap(path).post(data=data)
|
||||||
|
|
||||||
|
|
||||||
def index_new_video(
|
def index_new_video(youtube_id, video_type=VideoTypeEnum.VIDEOS):
|
||||||
youtube_id, video_overwrites=False, video_type=VideoTypeEnum.VIDEOS
|
|
||||||
):
|
|
||||||
"""combined classes to create new video in index"""
|
"""combined classes to create new video in index"""
|
||||||
video = YoutubeVideo(
|
video = YoutubeVideo(youtube_id, video_type=video_type)
|
||||||
youtube_id, video_overwrites=video_overwrites, video_type=video_type
|
|
||||||
)
|
|
||||||
video.build_json()
|
video.build_json()
|
||||||
if not video.json_data:
|
if not video.json_data:
|
||||||
raise ValueError("failed to get metadata for " + youtube_id)
|
raise ValueError("failed to get metadata for " + youtube_id)
|
||||||
|
|
|
@ -0,0 +1,81 @@
|
||||||
|
"""extract metadata from video streams"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import subprocess
|
||||||
|
from os import stat
|
||||||
|
|
||||||
|
|
||||||
|
class MediaStreamExtractor:
|
||||||
|
"""extract stream metadata"""
|
||||||
|
|
||||||
|
def __init__(self, media_path):
|
||||||
|
self.media_path = media_path
|
||||||
|
self.metadata = []
|
||||||
|
|
||||||
|
def extract_metadata(self):
|
||||||
|
"""entry point to extract metadata"""
|
||||||
|
|
||||||
|
cmd = [
|
||||||
|
"ffprobe",
|
||||||
|
"-v",
|
||||||
|
"quiet",
|
||||||
|
"-print_format",
|
||||||
|
"json",
|
||||||
|
"-show_streams",
|
||||||
|
"-show_format",
|
||||||
|
self.media_path,
|
||||||
|
]
|
||||||
|
result = subprocess.run(
|
||||||
|
cmd, capture_output=True, text=True, check=False
|
||||||
|
)
|
||||||
|
|
||||||
|
if result.returncode != 0:
|
||||||
|
return self.metadata
|
||||||
|
|
||||||
|
streams = json.loads(result.stdout).get("streams")
|
||||||
|
for stream in streams:
|
||||||
|
self.process_stream(stream)
|
||||||
|
|
||||||
|
return self.metadata
|
||||||
|
|
||||||
|
def process_stream(self, stream):
|
||||||
|
"""parse stream to metadata"""
|
||||||
|
codec_type = stream.get("codec_type")
|
||||||
|
if codec_type == "video":
|
||||||
|
self._extract_video_metadata(stream)
|
||||||
|
elif codec_type == "audio":
|
||||||
|
self._extract_audio_metadata(stream)
|
||||||
|
else:
|
||||||
|
return
|
||||||
|
|
||||||
|
def _extract_video_metadata(self, stream):
|
||||||
|
"""parse video metadata"""
|
||||||
|
if "bit_rate" not in stream:
|
||||||
|
# is probably thumbnail
|
||||||
|
return
|
||||||
|
|
||||||
|
self.metadata.append(
|
||||||
|
{
|
||||||
|
"type": "video",
|
||||||
|
"index": stream["index"],
|
||||||
|
"codec": stream["codec_name"],
|
||||||
|
"width": stream["width"],
|
||||||
|
"height": stream["height"],
|
||||||
|
"bitrate": int(stream["bit_rate"]),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
def _extract_audio_metadata(self, stream):
|
||||||
|
"""extract audio metadata"""
|
||||||
|
self.metadata.append(
|
||||||
|
{
|
||||||
|
"type": "audio",
|
||||||
|
"index": stream["index"],
|
||||||
|
"codec": stream.get("codec_name", "undefined"),
|
||||||
|
"bitrate": int(stream.get("bit_rate", 0)),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
def get_file_size(self):
|
||||||
|
"""get filesize in bytes"""
|
||||||
|
return stat(self.media_path).st_size
|
|
@ -0,0 +1,10 @@
|
||||||
|
from django.conf import settings
|
||||||
|
from django.contrib.auth.middleware import PersistentRemoteUserMiddleware
|
||||||
|
|
||||||
|
|
||||||
|
class HttpRemoteUserMiddleware(PersistentRemoteUserMiddleware):
|
||||||
|
"""This class allows authentication via HTTP_REMOTE_USER which is set for
|
||||||
|
example by certain SSO applications.
|
||||||
|
"""
|
||||||
|
|
||||||
|
header = settings.TA_AUTH_PROXY_USERNAME_HEADER
|
|
@ -5,23 +5,19 @@ Functionality:
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import json
|
import json
|
||||||
import os
|
|
||||||
import re
|
|
||||||
from random import randint
|
from random import randint
|
||||||
|
from time import sleep
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
from celery.schedules import crontab
|
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
from home.src.ta.ta_redis import RedisArchivist
|
from home.src.ta.ta_redis import RedisArchivist
|
||||||
|
|
||||||
|
|
||||||
class AppConfig:
|
class AppConfig:
|
||||||
"""handle user settings and application variables"""
|
"""handle application variables"""
|
||||||
|
|
||||||
def __init__(self, user_id=False):
|
def __init__(self):
|
||||||
self.user_id = user_id
|
|
||||||
self.config = self.get_config()
|
self.config = self.get_config()
|
||||||
self.colors = self.get_colors()
|
|
||||||
|
|
||||||
def get_config(self):
|
def get_config(self):
|
||||||
"""get config from default file or redis if changed"""
|
"""get config from default file or redis if changed"""
|
||||||
|
@ -29,13 +25,6 @@ class AppConfig:
|
||||||
if not config:
|
if not config:
|
||||||
config = self.get_config_file()
|
config = self.get_config_file()
|
||||||
|
|
||||||
if self.user_id:
|
|
||||||
key = f"{self.user_id}:page_size"
|
|
||||||
page_size = RedisArchivist().get_message(key)["status"]
|
|
||||||
if page_size:
|
|
||||||
config["archive"]["page_size"] = page_size
|
|
||||||
|
|
||||||
config["application"].update(self.get_config_env())
|
|
||||||
return config
|
return config
|
||||||
|
|
||||||
def get_config_file(self):
|
def get_config_file(self):
|
||||||
|
@ -43,35 +32,24 @@ class AppConfig:
|
||||||
with open("home/config.json", "r", encoding="utf-8") as f:
|
with open("home/config.json", "r", encoding="utf-8") as f:
|
||||||
config_file = json.load(f)
|
config_file = json.load(f)
|
||||||
|
|
||||||
config_file["application"].update(self.get_config_env())
|
|
||||||
|
|
||||||
return config_file
|
return config_file
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def get_config_env():
|
|
||||||
"""read environment application variables"""
|
|
||||||
es_pass = os.environ.get("ELASTIC_PASSWORD")
|
|
||||||
es_user = os.environ.get("ELASTIC_USER", default="elastic")
|
|
||||||
|
|
||||||
application = {
|
|
||||||
"REDIS_HOST": os.environ.get("REDIS_HOST"),
|
|
||||||
"es_url": os.environ.get("ES_URL"),
|
|
||||||
"es_auth": (es_user, es_pass),
|
|
||||||
"HOST_UID": int(os.environ.get("HOST_UID", False)),
|
|
||||||
"HOST_GID": int(os.environ.get("HOST_GID", False)),
|
|
||||||
"enable_cast": bool(os.environ.get("ENABLE_CAST")),
|
|
||||||
}
|
|
||||||
|
|
||||||
return application
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def get_config_redis():
|
def get_config_redis():
|
||||||
"""read config json set from redis to overwrite defaults"""
|
"""read config json set from redis to overwrite defaults"""
|
||||||
config = RedisArchivist().get_message("config")
|
for i in range(10):
|
||||||
if not list(config.values())[0]:
|
try:
|
||||||
return False
|
config = RedisArchivist().get_message("config")
|
||||||
|
if not list(config.values())[0]:
|
||||||
|
return False
|
||||||
|
|
||||||
return config
|
return config
|
||||||
|
|
||||||
|
except Exception: # pylint: disable=broad-except
|
||||||
|
print(f"... Redis connection failed, retry [{i}/10]")
|
||||||
|
sleep(3)
|
||||||
|
|
||||||
|
raise ConnectionError("failed to connect to redis")
|
||||||
|
|
||||||
def update_config(self, form_post):
|
def update_config(self, form_post):
|
||||||
"""update config values from settings form"""
|
"""update config values from settings form"""
|
||||||
|
@ -91,42 +69,9 @@ class AppConfig:
|
||||||
self.config[config_dict][config_value] = to_write
|
self.config[config_dict][config_value] = to_write
|
||||||
updated.append((config_value, to_write))
|
updated.append((config_value, to_write))
|
||||||
|
|
||||||
RedisArchivist().set_message("config", self.config)
|
RedisArchivist().set_message("config", self.config, save=True)
|
||||||
return updated
|
return updated
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def set_user_config(form_post, user_id):
|
|
||||||
"""set values in redis for user settings"""
|
|
||||||
for key, value in form_post.items():
|
|
||||||
if not value:
|
|
||||||
continue
|
|
||||||
|
|
||||||
message = {"status": value}
|
|
||||||
redis_key = f"{user_id}:{key}"
|
|
||||||
RedisArchivist().set_message(redis_key, message)
|
|
||||||
|
|
||||||
def get_colors(self):
|
|
||||||
"""overwrite config if user has set custom values"""
|
|
||||||
colors = False
|
|
||||||
if self.user_id:
|
|
||||||
col_dict = RedisArchivist().get_message(f"{self.user_id}:colors")
|
|
||||||
colors = col_dict["status"]
|
|
||||||
|
|
||||||
if not colors:
|
|
||||||
colors = self.config["application"]["colors"]
|
|
||||||
|
|
||||||
self.config["application"]["colors"] = colors
|
|
||||||
return colors
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _build_rand_daily():
|
|
||||||
"""build random daily schedule per installation"""
|
|
||||||
return {
|
|
||||||
"minute": randint(0, 59),
|
|
||||||
"hour": randint(0, 23),
|
|
||||||
"day_of_week": "*",
|
|
||||||
}
|
|
||||||
|
|
||||||
def load_new_defaults(self):
|
def load_new_defaults(self):
|
||||||
"""check config.json for missing defaults"""
|
"""check config.json for missing defaults"""
|
||||||
default_config = self.get_config_file()
|
default_config = self.get_config_file()
|
||||||
|
@ -135,7 +80,6 @@ class AppConfig:
|
||||||
# check for customizations
|
# check for customizations
|
||||||
if not redis_config:
|
if not redis_config:
|
||||||
config = self.get_config()
|
config = self.get_config()
|
||||||
config["scheduler"]["version_check"] = self._build_rand_daily()
|
|
||||||
RedisArchivist().set_message("config", config)
|
RedisArchivist().set_message("config", config)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
@ -151,9 +95,6 @@ class AppConfig:
|
||||||
# missing nested values
|
# missing nested values
|
||||||
for sub_key, sub_value in value.items():
|
for sub_key, sub_value in value.items():
|
||||||
if sub_key not in redis_config[key].keys():
|
if sub_key not in redis_config[key].keys():
|
||||||
if sub_value == "rand-d":
|
|
||||||
sub_value = self._build_rand_daily()
|
|
||||||
|
|
||||||
redis_config[key].update({sub_key: sub_value})
|
redis_config[key].update({sub_key: sub_value})
|
||||||
needs_update = True
|
needs_update = True
|
||||||
|
|
||||||
|
@ -163,201 +104,80 @@ class AppConfig:
|
||||||
return needs_update
|
return needs_update
|
||||||
|
|
||||||
|
|
||||||
class ScheduleBuilder:
|
|
||||||
"""build schedule dicts for beat"""
|
|
||||||
|
|
||||||
SCHEDULES = {
|
|
||||||
"update_subscribed": "0 8 *",
|
|
||||||
"download_pending": "0 16 *",
|
|
||||||
"check_reindex": "0 12 *",
|
|
||||||
"thumbnail_check": "0 17 *",
|
|
||||||
"run_backup": "0 18 0",
|
|
||||||
"version_check": "0 11 *",
|
|
||||||
}
|
|
||||||
CONFIG = ["check_reindex_days", "run_backup_rotate"]
|
|
||||||
MSG = "message:setting"
|
|
||||||
|
|
||||||
def __init__(self):
|
|
||||||
self.config = AppConfig().config
|
|
||||||
|
|
||||||
def update_schedule_conf(self, form_post):
|
|
||||||
"""process form post"""
|
|
||||||
print("processing form, restart container for changes to take effect")
|
|
||||||
redis_config = self.config
|
|
||||||
for key, value in form_post.items():
|
|
||||||
if key in self.SCHEDULES and value:
|
|
||||||
try:
|
|
||||||
to_write = self.value_builder(key, value)
|
|
||||||
except ValueError:
|
|
||||||
print(f"failed: {key} {value}")
|
|
||||||
mess_dict = {
|
|
||||||
"status": self.MSG,
|
|
||||||
"level": "error",
|
|
||||||
"title": "Scheduler update failed.",
|
|
||||||
"message": "Invalid schedule input",
|
|
||||||
}
|
|
||||||
RedisArchivist().set_message(
|
|
||||||
self.MSG, mess_dict, expire=True
|
|
||||||
)
|
|
||||||
return
|
|
||||||
|
|
||||||
redis_config["scheduler"][key] = to_write
|
|
||||||
if key in self.CONFIG and value:
|
|
||||||
redis_config["scheduler"][key] = int(value)
|
|
||||||
RedisArchivist().set_message("config", redis_config)
|
|
||||||
mess_dict = {
|
|
||||||
"status": self.MSG,
|
|
||||||
"level": "info",
|
|
||||||
"title": "Scheduler changed.",
|
|
||||||
"message": "Please restart container for changes to take effect",
|
|
||||||
}
|
|
||||||
RedisArchivist().set_message(self.MSG, mess_dict, expire=True)
|
|
||||||
|
|
||||||
def value_builder(self, key, value):
|
|
||||||
"""validate single cron form entry and return cron dict"""
|
|
||||||
print(f"change schedule for {key} to {value}")
|
|
||||||
if value == "0":
|
|
||||||
# deactivate this schedule
|
|
||||||
return False
|
|
||||||
if re.search(r"[\d]{1,2}\/[\d]{1,2}", value):
|
|
||||||
# number/number cron format will fail in celery
|
|
||||||
print("number/number schedule formatting not supported")
|
|
||||||
raise ValueError
|
|
||||||
|
|
||||||
keys = ["minute", "hour", "day_of_week"]
|
|
||||||
if value == "auto":
|
|
||||||
# set to sensible default
|
|
||||||
values = self.SCHEDULES[key].split()
|
|
||||||
else:
|
|
||||||
values = value.split()
|
|
||||||
|
|
||||||
if len(keys) != len(values):
|
|
||||||
print(f"failed to parse {value} for {key}")
|
|
||||||
raise ValueError("invalid input")
|
|
||||||
|
|
||||||
to_write = dict(zip(keys, values))
|
|
||||||
self._validate_cron(to_write)
|
|
||||||
|
|
||||||
return to_write
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _validate_cron(to_write):
|
|
||||||
"""validate all fields, raise value error for impossible schedule"""
|
|
||||||
all_hours = list(re.split(r"\D+", to_write["hour"]))
|
|
||||||
for hour in all_hours:
|
|
||||||
if hour.isdigit() and int(hour) > 23:
|
|
||||||
print("hour can not be greater than 23")
|
|
||||||
raise ValueError("invalid input")
|
|
||||||
|
|
||||||
all_days = list(re.split(r"\D+", to_write["day_of_week"]))
|
|
||||||
for day in all_days:
|
|
||||||
if day.isdigit() and int(day) > 6:
|
|
||||||
print("day can not be greater than 6")
|
|
||||||
raise ValueError("invalid input")
|
|
||||||
|
|
||||||
if not to_write["minute"].isdigit():
|
|
||||||
print("too frequent: only number in minutes are supported")
|
|
||||||
raise ValueError("invalid input")
|
|
||||||
|
|
||||||
if int(to_write["minute"]) > 59:
|
|
||||||
print("minutes can not be greater than 59")
|
|
||||||
raise ValueError("invalid input")
|
|
||||||
|
|
||||||
def build_schedule(self):
|
|
||||||
"""build schedule dict as expected by app.conf.beat_schedule"""
|
|
||||||
AppConfig().load_new_defaults()
|
|
||||||
self.config = AppConfig().config
|
|
||||||
schedule_dict = {}
|
|
||||||
|
|
||||||
for schedule_item in self.SCHEDULES:
|
|
||||||
item_conf = self.config["scheduler"][schedule_item]
|
|
||||||
if not item_conf:
|
|
||||||
continue
|
|
||||||
|
|
||||||
schedule_dict.update(
|
|
||||||
{
|
|
||||||
f"schedule_{schedule_item}": {
|
|
||||||
"task": schedule_item,
|
|
||||||
"schedule": crontab(
|
|
||||||
minute=item_conf["minute"],
|
|
||||||
hour=item_conf["hour"],
|
|
||||||
day_of_week=item_conf["day_of_week"],
|
|
||||||
),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
return schedule_dict
|
|
||||||
|
|
||||||
|
|
||||||
class ReleaseVersion:
|
class ReleaseVersion:
|
||||||
"""compare local version with remote version"""
|
"""compare local version with remote version"""
|
||||||
|
|
||||||
REMOTE_URL = "https://www.tubearchivist.com/api/release/latest/"
|
REMOTE_URL = "https://www.tubearchivist.com/api/release/latest/"
|
||||||
NEW_KEY = "versioncheck:new"
|
NEW_KEY = "versioncheck:new"
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self) -> None:
|
||||||
self.local_version = self._parse_version(settings.TA_VERSION)
|
self.local_version: str = settings.TA_VERSION
|
||||||
self.is_unstable = settings.TA_VERSION.endswith("-unstable")
|
self.is_unstable: bool = settings.TA_VERSION.endswith("-unstable")
|
||||||
self.remote_version = False
|
self.remote_version: str = ""
|
||||||
self.is_breaking = False
|
self.is_breaking: bool = False
|
||||||
self.response = False
|
|
||||||
|
|
||||||
def check(self):
|
def check(self) -> None:
|
||||||
"""check version"""
|
"""check version"""
|
||||||
print(f"[{self.local_version}]: look for updates")
|
print(f"[{self.local_version}]: look for updates")
|
||||||
self.get_remote_version()
|
self.get_remote_version()
|
||||||
new_version, is_breaking = self._has_update()
|
new_version = self._has_update()
|
||||||
if new_version:
|
if new_version:
|
||||||
message = {
|
message = {
|
||||||
"status": True,
|
"status": True,
|
||||||
"version": new_version,
|
"version": new_version,
|
||||||
"is_breaking": is_breaking,
|
"is_breaking": self.is_breaking,
|
||||||
}
|
}
|
||||||
RedisArchivist().set_message(self.NEW_KEY, message)
|
RedisArchivist().set_message(self.NEW_KEY, message)
|
||||||
print(f"[{self.local_version}]: found new version {new_version}")
|
print(f"[{self.local_version}]: found new version {new_version}")
|
||||||
|
|
||||||
def get_remote_version(self):
|
def get_local_version(self) -> str:
|
||||||
|
"""read version from local"""
|
||||||
|
return self.local_version
|
||||||
|
|
||||||
|
def get_remote_version(self) -> None:
|
||||||
"""read version from remote"""
|
"""read version from remote"""
|
||||||
self.response = requests.get(self.REMOTE_URL, timeout=20).json()
|
sleep(randint(0, 60))
|
||||||
remote_version_str = self.response["release_version"]
|
response = requests.get(self.REMOTE_URL, timeout=20).json()
|
||||||
self.remote_version = self._parse_version(remote_version_str)
|
self.remote_version = response["release_version"]
|
||||||
self.is_breaking = self.response["breaking_changes"]
|
self.is_breaking = response["breaking_changes"]
|
||||||
|
|
||||||
def _has_update(self):
|
def _has_update(self) -> str | bool:
|
||||||
"""check if there is an update"""
|
"""check if there is an update"""
|
||||||
for idx, number in enumerate(self.local_version):
|
remote_parsed = self._parse_version(self.remote_version)
|
||||||
is_newer = self.remote_version[idx] > number
|
local_parsed = self._parse_version(self.local_version)
|
||||||
if is_newer:
|
if remote_parsed > local_parsed:
|
||||||
return self.response["release_version"], self.is_breaking
|
return self.remote_version
|
||||||
|
|
||||||
if self.is_unstable and self.local_version == self.remote_version:
|
if self.is_unstable and local_parsed == remote_parsed:
|
||||||
return self.response["release_version"], self.is_breaking
|
return self.remote_version
|
||||||
|
|
||||||
return False, False
|
return False
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _parse_version(version):
|
def _parse_version(version) -> tuple[int, ...]:
|
||||||
"""return version parts"""
|
"""return version parts"""
|
||||||
clean = version.rstrip("-unstable").lstrip("v")
|
clean = version.rstrip("-unstable").lstrip("v")
|
||||||
return tuple((int(i) for i in clean.split(".")))
|
return tuple((int(i) for i in clean.split(".")))
|
||||||
|
|
||||||
def is_updated(self):
|
def is_updated(self) -> str | bool:
|
||||||
"""check if update happened in the mean time"""
|
"""check if update happened in the mean time"""
|
||||||
message = self.get_update()
|
message = self.get_update()
|
||||||
if not message:
|
if not message:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
if self._parse_version(message.get("version")) == self.local_version:
|
local_parsed = self._parse_version(self.local_version)
|
||||||
|
message_parsed = self._parse_version(message.get("version"))
|
||||||
|
|
||||||
|
if local_parsed >= message_parsed:
|
||||||
RedisArchivist().del_message(self.NEW_KEY)
|
RedisArchivist().del_message(self.NEW_KEY)
|
||||||
return settings.TA_VERSION
|
return settings.TA_VERSION
|
||||||
|
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def get_update(self):
|
def get_update(self) -> dict:
|
||||||
"""return new version dict if available"""
|
"""return new version dict if available"""
|
||||||
message = RedisArchivist().get_message(self.NEW_KEY)
|
message = RedisArchivist().get_message(self.NEW_KEY)
|
||||||
if not message.get("status"):
|
if not message.get("status"):
|
||||||
return False
|
return {}
|
||||||
|
|
||||||
return message
|
return message
|
||||||
|
|
|
@ -0,0 +1,89 @@
|
||||||
|
"""
|
||||||
|
Functionality:
|
||||||
|
- Handle scheduler config update
|
||||||
|
"""
|
||||||
|
|
||||||
|
from django_celery_beat.models import CrontabSchedule
|
||||||
|
from home.models import CustomPeriodicTask
|
||||||
|
from home.src.ta.config import AppConfig
|
||||||
|
from home.src.ta.settings import EnvironmentSettings
|
||||||
|
from home.src.ta.task_config import TASK_CONFIG
|
||||||
|
|
||||||
|
|
||||||
|
class ScheduleBuilder:
|
||||||
|
"""build schedule dicts for beat"""
|
||||||
|
|
||||||
|
SCHEDULES = {
|
||||||
|
"update_subscribed": "0 8 *",
|
||||||
|
"download_pending": "0 16 *",
|
||||||
|
"check_reindex": "0 12 *",
|
||||||
|
"thumbnail_check": "0 17 *",
|
||||||
|
"run_backup": "0 18 0",
|
||||||
|
"version_check": "0 11 *",
|
||||||
|
}
|
||||||
|
CONFIG = {
|
||||||
|
"check_reindex_days": "check_reindex",
|
||||||
|
"run_backup_rotate": "run_backup",
|
||||||
|
"update_subscribed_notify": "update_subscribed",
|
||||||
|
"download_pending_notify": "download_pending",
|
||||||
|
"check_reindex_notify": "check_reindex",
|
||||||
|
}
|
||||||
|
MSG = "message:setting"
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.config = AppConfig().config
|
||||||
|
|
||||||
|
def update_schedule_conf(self, form_post):
|
||||||
|
"""process form post, schedules need to be validated before"""
|
||||||
|
for key, value in form_post.items():
|
||||||
|
if not value:
|
||||||
|
continue
|
||||||
|
|
||||||
|
if key in self.SCHEDULES:
|
||||||
|
if value == "auto":
|
||||||
|
value = self.SCHEDULES.get(key)
|
||||||
|
|
||||||
|
_ = self.get_set_task(key, value)
|
||||||
|
continue
|
||||||
|
|
||||||
|
if key in self.CONFIG:
|
||||||
|
self.set_config(key, value)
|
||||||
|
|
||||||
|
def get_set_task(self, task_name, schedule=False):
|
||||||
|
"""get task"""
|
||||||
|
try:
|
||||||
|
task = CustomPeriodicTask.objects.get(name=task_name)
|
||||||
|
except CustomPeriodicTask.DoesNotExist:
|
||||||
|
description = TASK_CONFIG[task_name].get("title")
|
||||||
|
task = CustomPeriodicTask(
|
||||||
|
name=task_name,
|
||||||
|
task=task_name,
|
||||||
|
description=description,
|
||||||
|
)
|
||||||
|
|
||||||
|
if schedule:
|
||||||
|
task_crontab = self.get_set_cron_tab(schedule)
|
||||||
|
task.crontab = task_crontab
|
||||||
|
task.save()
|
||||||
|
|
||||||
|
return task
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def get_set_cron_tab(schedule):
|
||||||
|
"""needs to be validated before"""
|
||||||
|
kwargs = dict(zip(["minute", "hour", "day_of_week"], schedule.split()))
|
||||||
|
kwargs.update({"timezone": EnvironmentSettings.TZ})
|
||||||
|
crontab, _ = CrontabSchedule.objects.get_or_create(**kwargs)
|
||||||
|
|
||||||
|
return crontab
|
||||||
|
|
||||||
|
def set_config(self, key, value):
|
||||||
|
"""set task_config"""
|
||||||
|
task_name = self.CONFIG.get(key)
|
||||||
|
if not task_name:
|
||||||
|
raise ValueError("invalid config key")
|
||||||
|
|
||||||
|
task = CustomPeriodicTask.objects.get(name=task_name)
|
||||||
|
config_key = key.split(f"{task_name}_")[-1]
|
||||||
|
task.task_config.update({config_key: value})
|
||||||
|
task.save()
|
|
@ -0,0 +1,11 @@
|
||||||
|
from django.http import HttpResponse
|
||||||
|
|
||||||
|
|
||||||
|
class HealthCheckMiddleware:
|
||||||
|
def __init__(self, get_response):
|
||||||
|
self.get_response = get_response
|
||||||
|
|
||||||
|
def __call__(self, request):
|
||||||
|
if request.path == "/health":
|
||||||
|
return HttpResponse("ok")
|
||||||
|
return self.get_response(request)
|
|
@ -6,30 +6,26 @@ Loose collection of helper functions
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import random
|
import random
|
||||||
import re
|
|
||||||
import string
|
import string
|
||||||
import subprocess
|
import subprocess
|
||||||
import unicodedata
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
from typing import Any
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
|
from home.src.es.connect import IndexPaginate
|
||||||
|
from home.src.ta.settings import EnvironmentSettings
|
||||||
|
|
||||||
|
|
||||||
def clean_string(file_name):
|
def ignore_filelist(filelist: list[str]) -> list[str]:
|
||||||
"""clean string to only asci characters"""
|
|
||||||
whitelist = "-_.() " + string.ascii_letters + string.digits
|
|
||||||
normalized = unicodedata.normalize("NFKD", file_name)
|
|
||||||
ascii_only = normalized.encode("ASCII", "ignore").decode().strip()
|
|
||||||
white_listed = "".join(c for c in ascii_only if c in whitelist)
|
|
||||||
cleaned = re.sub(r"[ ]{2,}", " ", white_listed)
|
|
||||||
return cleaned
|
|
||||||
|
|
||||||
|
|
||||||
def ignore_filelist(filelist):
|
|
||||||
"""ignore temp files for os.listdir sanitizer"""
|
"""ignore temp files for os.listdir sanitizer"""
|
||||||
to_ignore = ["Icon\r\r", "Temporary Items", "Network Trash Folder"]
|
to_ignore = [
|
||||||
cleaned = []
|
"@eaDir",
|
||||||
|
"Icon\r\r",
|
||||||
|
"Network Trash Folder",
|
||||||
|
"Temporary Items",
|
||||||
|
]
|
||||||
|
cleaned: list[str] = []
|
||||||
for file_name in filelist:
|
for file_name in filelist:
|
||||||
if file_name.startswith(".") or file_name in to_ignore:
|
if file_name.startswith(".") or file_name in to_ignore:
|
||||||
continue
|
continue
|
||||||
|
@ -39,13 +35,13 @@ def ignore_filelist(filelist):
|
||||||
return cleaned
|
return cleaned
|
||||||
|
|
||||||
|
|
||||||
def randomizor(length):
|
def randomizor(length: int) -> str:
|
||||||
"""generate random alpha numeric string"""
|
"""generate random alpha numeric string"""
|
||||||
pool = string.digits + string.ascii_letters
|
pool: str = string.digits + string.ascii_letters
|
||||||
return "".join(random.choice(pool) for i in range(length))
|
return "".join(random.choice(pool) for i in range(length))
|
||||||
|
|
||||||
|
|
||||||
def requests_headers():
|
def requests_headers() -> dict[str, str]:
|
||||||
"""build header with random user agent for requests outside of yt-dlp"""
|
"""build header with random user agent for requests outside of yt-dlp"""
|
||||||
|
|
||||||
chrome_versions = (
|
chrome_versions = (
|
||||||
|
@ -97,17 +93,17 @@ def requests_headers():
|
||||||
return {"User-Agent": template}
|
return {"User-Agent": template}
|
||||||
|
|
||||||
|
|
||||||
def date_praser(timestamp):
|
def date_praser(timestamp: int | str) -> str:
|
||||||
"""return formatted date string"""
|
"""return formatted date string"""
|
||||||
if isinstance(timestamp, int):
|
if isinstance(timestamp, int):
|
||||||
date_obj = datetime.fromtimestamp(timestamp)
|
date_obj = datetime.fromtimestamp(timestamp)
|
||||||
elif isinstance(timestamp, str):
|
elif isinstance(timestamp, str):
|
||||||
date_obj = datetime.strptime(timestamp, "%Y-%m-%d")
|
date_obj = datetime.strptime(timestamp, "%Y-%m-%d")
|
||||||
|
|
||||||
return datetime.strftime(date_obj, "%d %b, %Y")
|
return date_obj.date().isoformat()
|
||||||
|
|
||||||
|
|
||||||
def time_parser(timestamp):
|
def time_parser(timestamp: str) -> float:
|
||||||
"""return seconds from timestamp, false on empty"""
|
"""return seconds from timestamp, false on empty"""
|
||||||
if not timestamp:
|
if not timestamp:
|
||||||
return False
|
return False
|
||||||
|
@ -119,27 +115,27 @@ def time_parser(timestamp):
|
||||||
return int(hours) * 60 * 60 + int(minutes) * 60 + float(seconds)
|
return int(hours) * 60 * 60 + int(minutes) * 60 + float(seconds)
|
||||||
|
|
||||||
|
|
||||||
def clear_dl_cache(config):
|
def clear_dl_cache(cache_dir: str) -> int:
|
||||||
"""clear leftover files from dl cache"""
|
"""clear leftover files from dl cache"""
|
||||||
print("clear download cache")
|
print("clear download cache")
|
||||||
cache_dir = os.path.join(config["application"]["cache_dir"], "download")
|
download_cache_dir = os.path.join(cache_dir, "download")
|
||||||
leftover_files = os.listdir(cache_dir)
|
leftover_files = ignore_filelist(os.listdir(download_cache_dir))
|
||||||
for cached in leftover_files:
|
for cached in leftover_files:
|
||||||
to_delete = os.path.join(cache_dir, cached)
|
to_delete = os.path.join(download_cache_dir, cached)
|
||||||
os.remove(to_delete)
|
os.remove(to_delete)
|
||||||
|
|
||||||
return len(leftover_files)
|
return len(leftover_files)
|
||||||
|
|
||||||
|
|
||||||
def get_mapping():
|
def get_mapping() -> dict:
|
||||||
"""read index_mapping.json and get expected mapping and settings"""
|
"""read index_mapping.json and get expected mapping and settings"""
|
||||||
with open("home/src/es/index_mapping.json", "r", encoding="utf-8") as f:
|
with open("home/src/es/index_mapping.json", "r", encoding="utf-8") as f:
|
||||||
index_config = json.load(f).get("index_config")
|
index_config: dict = json.load(f).get("index_config")
|
||||||
|
|
||||||
return index_config
|
return index_config
|
||||||
|
|
||||||
|
|
||||||
def is_shorts(youtube_id):
|
def is_shorts(youtube_id: str) -> bool:
|
||||||
"""check if youtube_id is a shorts video, bot not it it's not a shorts"""
|
"""check if youtube_id is a shorts video, bot not it it's not a shorts"""
|
||||||
shorts_url = f"https://www.youtube.com/shorts/{youtube_id}"
|
shorts_url = f"https://www.youtube.com/shorts/{youtube_id}"
|
||||||
response = requests.head(
|
response = requests.head(
|
||||||
|
@ -149,10 +145,57 @@ def is_shorts(youtube_id):
|
||||||
return response.status_code == 200
|
return response.status_code == 200
|
||||||
|
|
||||||
|
|
||||||
def ta_host_parser(ta_host):
|
def get_duration_sec(file_path: str) -> int:
|
||||||
|
"""get duration of media file from file path"""
|
||||||
|
|
||||||
|
duration = subprocess.run(
|
||||||
|
[
|
||||||
|
"ffprobe",
|
||||||
|
"-v",
|
||||||
|
"error",
|
||||||
|
"-show_entries",
|
||||||
|
"format=duration",
|
||||||
|
"-of",
|
||||||
|
"default=noprint_wrappers=1:nokey=1",
|
||||||
|
file_path,
|
||||||
|
],
|
||||||
|
capture_output=True,
|
||||||
|
check=True,
|
||||||
|
)
|
||||||
|
duration_raw = duration.stdout.decode().strip()
|
||||||
|
if duration_raw == "N/A":
|
||||||
|
return 0
|
||||||
|
|
||||||
|
duration_sec = int(float(duration_raw))
|
||||||
|
return duration_sec
|
||||||
|
|
||||||
|
|
||||||
|
def get_duration_str(seconds: int) -> str:
|
||||||
|
"""Return a human-readable duration string from seconds."""
|
||||||
|
if not seconds:
|
||||||
|
return "NA"
|
||||||
|
|
||||||
|
units = [("y", 31536000), ("d", 86400), ("h", 3600), ("m", 60), ("s", 1)]
|
||||||
|
duration_parts = []
|
||||||
|
|
||||||
|
for unit_label, unit_seconds in units:
|
||||||
|
if seconds >= unit_seconds:
|
||||||
|
unit_count, seconds = divmod(seconds, unit_seconds)
|
||||||
|
duration_parts.append(f"{unit_count:02}{unit_label}")
|
||||||
|
|
||||||
|
return " ".join(duration_parts)
|
||||||
|
|
||||||
|
|
||||||
|
def ta_host_parser(ta_host: str) -> tuple[list[str], list[str]]:
|
||||||
"""parse ta_host env var for ALLOWED_HOSTS and CSRF_TRUSTED_ORIGINS"""
|
"""parse ta_host env var for ALLOWED_HOSTS and CSRF_TRUSTED_ORIGINS"""
|
||||||
allowed_hosts = []
|
allowed_hosts: list[str] = [
|
||||||
csrf_trusted_origins = []
|
"localhost",
|
||||||
|
"tubearchivist",
|
||||||
|
]
|
||||||
|
csrf_trusted_origins: list[str] = [
|
||||||
|
"http://localhost",
|
||||||
|
"http://tubearchivist",
|
||||||
|
]
|
||||||
for host in ta_host.split():
|
for host in ta_host.split():
|
||||||
host_clean = host.strip()
|
host_clean = host.strip()
|
||||||
if not host_clean.startswith("http"):
|
if not host_clean.startswith("http"):
|
||||||
|
@ -165,52 +208,53 @@ def ta_host_parser(ta_host):
|
||||||
return allowed_hosts, csrf_trusted_origins
|
return allowed_hosts, csrf_trusted_origins
|
||||||
|
|
||||||
|
|
||||||
class DurationConverter:
|
def get_stylesheets():
|
||||||
"""
|
"""Get all valid stylesheets from /static/css"""
|
||||||
using ffmpeg to get and parse duration from filepath
|
app_root = EnvironmentSettings.APP_DIR
|
||||||
"""
|
stylesheets = os.listdir(os.path.join(app_root, "static/css"))
|
||||||
|
stylesheets.remove("style.css")
|
||||||
|
stylesheets.sort()
|
||||||
|
stylesheets = list(filter(lambda x: x.endswith(".css"), stylesheets))
|
||||||
|
return stylesheets
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def get_sec(file_path):
|
|
||||||
"""read duration from file"""
|
|
||||||
duration = subprocess.run(
|
|
||||||
[
|
|
||||||
"ffprobe",
|
|
||||||
"-v",
|
|
||||||
"error",
|
|
||||||
"-show_entries",
|
|
||||||
"format=duration",
|
|
||||||
"-of",
|
|
||||||
"default=noprint_wrappers=1:nokey=1",
|
|
||||||
file_path,
|
|
||||||
],
|
|
||||||
capture_output=True,
|
|
||||||
check=True,
|
|
||||||
)
|
|
||||||
duration_raw = duration.stdout.decode().strip()
|
|
||||||
if duration_raw == "N/A":
|
|
||||||
return 0
|
|
||||||
|
|
||||||
duration_sec = int(float(duration_raw))
|
def check_stylesheet(stylesheet: str):
|
||||||
return duration_sec
|
"""Check if a stylesheet exists. Return dark.css as a fallback"""
|
||||||
|
if stylesheet in get_stylesheets():
|
||||||
|
return stylesheet
|
||||||
|
|
||||||
@staticmethod
|
return "dark.css"
|
||||||
def get_str(duration_sec):
|
|
||||||
"""takes duration in sec and returns clean string"""
|
|
||||||
if not duration_sec:
|
|
||||||
# failed to extract
|
|
||||||
return "NA"
|
|
||||||
|
|
||||||
hours = duration_sec // 3600
|
|
||||||
minutes = (duration_sec - (hours * 3600)) // 60
|
|
||||||
secs = duration_sec - (hours * 3600) - (minutes * 60)
|
|
||||||
|
|
||||||
duration_str = str()
|
def is_missing(
|
||||||
if hours:
|
to_check: str | list[str],
|
||||||
duration_str = str(hours).zfill(2) + ":"
|
index_name: str = "ta_video,ta_download",
|
||||||
if minutes:
|
on_key: str = "youtube_id",
|
||||||
duration_str = duration_str + str(minutes).zfill(2) + ":"
|
) -> list[str]:
|
||||||
else:
|
"""id or list of ids that are missing from index_name"""
|
||||||
duration_str = duration_str + "00:"
|
if isinstance(to_check, str):
|
||||||
duration_str = duration_str + str(secs).zfill(2)
|
to_check = [to_check]
|
||||||
return duration_str
|
|
||||||
|
data = {
|
||||||
|
"query": {"terms": {on_key: to_check}},
|
||||||
|
"_source": [on_key],
|
||||||
|
}
|
||||||
|
result = IndexPaginate(index_name, data=data).get_results()
|
||||||
|
existing_ids = [i[on_key] for i in result]
|
||||||
|
dl = [i for i in to_check if i not in existing_ids]
|
||||||
|
|
||||||
|
return dl
|
||||||
|
|
||||||
|
|
||||||
|
def get_channel_overwrites() -> dict[str, dict[str, Any]]:
|
||||||
|
"""get overwrites indexed my channel_id"""
|
||||||
|
data = {
|
||||||
|
"query": {
|
||||||
|
"bool": {"must": [{"exists": {"field": "channel_overwrites"}}]}
|
||||||
|
},
|
||||||
|
"_source": ["channel_id", "channel_overwrites"],
|
||||||
|
}
|
||||||
|
result = IndexPaginate("ta_channel", data).get_results()
|
||||||
|
overwrites = {i["channel_id"]: i["channel_overwrites"] for i in result}
|
||||||
|
|
||||||
|
return overwrites
|
||||||
|
|
|
@ -0,0 +1,141 @@
|
||||||
|
"""send notifications using apprise"""
|
||||||
|
|
||||||
|
import apprise
|
||||||
|
from home.src.es.connect import ElasticWrap
|
||||||
|
from home.src.ta.task_config import TASK_CONFIG
|
||||||
|
from home.src.ta.task_manager import TaskManager
|
||||||
|
|
||||||
|
|
||||||
|
class Notifications:
|
||||||
|
"""store notifications in ES"""
|
||||||
|
|
||||||
|
GET_PATH = "ta_config/_doc/notify"
|
||||||
|
UPDATE_PATH = "ta_config/_update/notify/"
|
||||||
|
|
||||||
|
def __init__(self, task_name: str):
|
||||||
|
self.task_name = task_name
|
||||||
|
|
||||||
|
def send(self, task_id: str, task_title: str) -> None:
|
||||||
|
"""send notifications"""
|
||||||
|
apobj = apprise.Apprise()
|
||||||
|
urls: list[str] = self.get_urls()
|
||||||
|
if not urls:
|
||||||
|
return
|
||||||
|
|
||||||
|
title, body = self._build_message(task_id, task_title)
|
||||||
|
|
||||||
|
if not body:
|
||||||
|
return
|
||||||
|
|
||||||
|
for url in urls:
|
||||||
|
apobj.add(url)
|
||||||
|
|
||||||
|
apobj.notify(body=body, title=title)
|
||||||
|
|
||||||
|
def _build_message(
|
||||||
|
self, task_id: str, task_title: str
|
||||||
|
) -> tuple[str, str | None]:
|
||||||
|
"""build message to send notification"""
|
||||||
|
task = TaskManager().get_task(task_id)
|
||||||
|
status = task.get("status")
|
||||||
|
title: str = f"[TA] {task_title} process ended with {status}"
|
||||||
|
body: str | None = task.get("result")
|
||||||
|
|
||||||
|
return title, body
|
||||||
|
|
||||||
|
def get_urls(self) -> list[str]:
|
||||||
|
"""get stored urls for task"""
|
||||||
|
response, code = ElasticWrap(self.GET_PATH).get(print_error=False)
|
||||||
|
if not code == 200:
|
||||||
|
return []
|
||||||
|
|
||||||
|
urls = response["_source"].get(self.task_name, [])
|
||||||
|
|
||||||
|
return urls
|
||||||
|
|
||||||
|
def add_url(self, url: str) -> None:
|
||||||
|
"""add url to task notification"""
|
||||||
|
source = (
|
||||||
|
"if (!ctx._source.containsKey(params.task_name)) "
|
||||||
|
+ "{ctx._source[params.task_name] = [params.url]} "
|
||||||
|
+ "else if (!ctx._source[params.task_name].contains(params.url)) "
|
||||||
|
+ "{ctx._source[params.task_name].add(params.url)} "
|
||||||
|
+ "else {ctx.op = 'none'}"
|
||||||
|
)
|
||||||
|
|
||||||
|
data = {
|
||||||
|
"script": {
|
||||||
|
"source": source,
|
||||||
|
"lang": "painless",
|
||||||
|
"params": {"url": url, "task_name": self.task_name},
|
||||||
|
},
|
||||||
|
"upsert": {self.task_name: [url]},
|
||||||
|
}
|
||||||
|
|
||||||
|
_, _ = ElasticWrap(self.UPDATE_PATH).post(data)
|
||||||
|
|
||||||
|
def remove_url(self, url: str) -> tuple[dict, int]:
|
||||||
|
"""remove url from task"""
|
||||||
|
source = (
|
||||||
|
"if (ctx._source.containsKey(params.task_name) "
|
||||||
|
+ "&& ctx._source[params.task_name].contains(params.url)) "
|
||||||
|
+ "{ctx._source[params.task_name]."
|
||||||
|
+ "remove(ctx._source[params.task_name].indexOf(params.url))}"
|
||||||
|
)
|
||||||
|
|
||||||
|
data = {
|
||||||
|
"script": {
|
||||||
|
"source": source,
|
||||||
|
"lang": "painless",
|
||||||
|
"params": {"url": url, "task_name": self.task_name},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
response, status_code = ElasticWrap(self.UPDATE_PATH).post(data)
|
||||||
|
if not self.get_urls():
|
||||||
|
_, _ = self.remove_task()
|
||||||
|
|
||||||
|
return response, status_code
|
||||||
|
|
||||||
|
def remove_task(self) -> tuple[dict, int]:
|
||||||
|
"""remove all notifications from task"""
|
||||||
|
source = (
|
||||||
|
"if (ctx._source.containsKey(params.task_name)) "
|
||||||
|
+ "{ctx._source.remove(params.task_name)}"
|
||||||
|
)
|
||||||
|
data = {
|
||||||
|
"script": {
|
||||||
|
"source": source,
|
||||||
|
"lang": "painless",
|
||||||
|
"params": {"task_name": self.task_name},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
response, status_code = ElasticWrap(self.UPDATE_PATH).post(data)
|
||||||
|
|
||||||
|
return response, status_code
|
||||||
|
|
||||||
|
|
||||||
|
def get_all_notifications() -> dict[str, list[str]]:
|
||||||
|
"""get all notifications stored"""
|
||||||
|
path = "ta_config/_doc/notify"
|
||||||
|
response, status_code = ElasticWrap(path).get(print_error=False)
|
||||||
|
if not status_code == 200:
|
||||||
|
return {}
|
||||||
|
|
||||||
|
notifications: dict = {}
|
||||||
|
source = response.get("_source")
|
||||||
|
if not source:
|
||||||
|
return notifications
|
||||||
|
|
||||||
|
for task_id, urls in source.items():
|
||||||
|
notifications.update(
|
||||||
|
{
|
||||||
|
task_id: {
|
||||||
|
"urls": urls,
|
||||||
|
"title": TASK_CONFIG[task_id]["title"],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
return notifications
|
|
@ -0,0 +1,95 @@
|
||||||
|
"""
|
||||||
|
Functionality:
|
||||||
|
- read and write application config backed by ES
|
||||||
|
- encapsulate persistence of application properties
|
||||||
|
"""
|
||||||
|
|
||||||
|
from os import environ
|
||||||
|
|
||||||
|
|
||||||
|
class EnvironmentSettings:
|
||||||
|
"""
|
||||||
|
Handle settings for the application that are driven from the environment.
|
||||||
|
These will not change when the user is using the application.
|
||||||
|
These settings are only provided only on startup.
|
||||||
|
"""
|
||||||
|
|
||||||
|
HOST_UID: int = int(environ.get("HOST_UID", False))
|
||||||
|
HOST_GID: int = int(environ.get("HOST_GID", False))
|
||||||
|
ENABLE_CAST: bool = bool(environ.get("ENABLE_CAST"))
|
||||||
|
TZ: str = str(environ.get("TZ", "UTC"))
|
||||||
|
TA_PORT: int = int(environ.get("TA_PORT", False))
|
||||||
|
TA_UWSGI_PORT: int = int(environ.get("TA_UWSGI_PORT", False))
|
||||||
|
TA_USERNAME: str = str(environ.get("TA_USERNAME"))
|
||||||
|
TA_PASSWORD: str = str(environ.get("TA_PASSWORD"))
|
||||||
|
|
||||||
|
# Application Paths
|
||||||
|
MEDIA_DIR: str = str(environ.get("TA_MEDIA_DIR", "/youtube"))
|
||||||
|
APP_DIR: str = str(environ.get("TA_APP_DIR", "/app"))
|
||||||
|
CACHE_DIR: str = str(environ.get("TA_CACHE_DIR", "/cache"))
|
||||||
|
|
||||||
|
# Redis
|
||||||
|
REDIS_HOST: str = str(environ.get("REDIS_HOST"))
|
||||||
|
REDIS_PORT: int = int(environ.get("REDIS_PORT", 6379))
|
||||||
|
REDIS_NAME_SPACE: str = str(environ.get("REDIS_NAME_SPACE", "ta:"))
|
||||||
|
|
||||||
|
# ElasticSearch
|
||||||
|
ES_URL: str = str(environ.get("ES_URL"))
|
||||||
|
ES_PASS: str = str(environ.get("ELASTIC_PASSWORD"))
|
||||||
|
ES_USER: str = str(environ.get("ELASTIC_USER", "elastic"))
|
||||||
|
ES_SNAPSHOT_DIR: str = str(
|
||||||
|
environ.get(
|
||||||
|
"ES_SNAPSHOT_DIR", "/usr/share/elasticsearch/data/snapshot"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
ES_DISABLE_VERIFY_SSL: bool = bool(environ.get("ES_DISABLE_VERIFY_SSL"))
|
||||||
|
|
||||||
|
def print_generic(self):
|
||||||
|
"""print generic env vars"""
|
||||||
|
print(
|
||||||
|
f"""
|
||||||
|
HOST_UID: {self.HOST_UID}
|
||||||
|
HOST_GID: {self.HOST_GID}
|
||||||
|
TZ: {self.TZ}
|
||||||
|
ENABLE_CAST: {self.ENABLE_CAST}
|
||||||
|
TA_PORT: {self.TA_PORT}
|
||||||
|
TA_UWSGI_PORT: {self.TA_UWSGI_PORT}
|
||||||
|
TA_USERNAME: {self.TA_USERNAME}
|
||||||
|
TA_PASSWORD: *****"""
|
||||||
|
)
|
||||||
|
|
||||||
|
def print_paths(self):
|
||||||
|
"""debug paths set"""
|
||||||
|
print(
|
||||||
|
f"""
|
||||||
|
MEDIA_DIR: {self.MEDIA_DIR}
|
||||||
|
APP_DIR: {self.APP_DIR}
|
||||||
|
CACHE_DIR: {self.CACHE_DIR}"""
|
||||||
|
)
|
||||||
|
|
||||||
|
def print_redis_conf(self):
|
||||||
|
"""debug redis conf paths"""
|
||||||
|
print(
|
||||||
|
f"""
|
||||||
|
REDIS_HOST: {self.REDIS_HOST}
|
||||||
|
REDIS_PORT: {self.REDIS_PORT}
|
||||||
|
REDIS_NAME_SPACE: {self.REDIS_NAME_SPACE}"""
|
||||||
|
)
|
||||||
|
|
||||||
|
def print_es_paths(self):
|
||||||
|
"""debug es conf"""
|
||||||
|
print(
|
||||||
|
f"""
|
||||||
|
ES_URL: {self.ES_URL}
|
||||||
|
ES_PASS: *****
|
||||||
|
ES_USER: {self.ES_USER}
|
||||||
|
ES_SNAPSHOT_DIR: {self.ES_SNAPSHOT_DIR}
|
||||||
|
ES_DISABLE_VERIFY_SSL: {self.ES_DISABLE_VERIFY_SSL}"""
|
||||||
|
)
|
||||||
|
|
||||||
|
def print_all(self):
|
||||||
|
"""print all"""
|
||||||
|
self.print_generic()
|
||||||
|
self.print_paths()
|
||||||
|
self.print_redis_conf()
|
||||||
|
self.print_es_paths()
|
|
@ -6,20 +6,22 @@ functionality:
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import json
|
import json
|
||||||
import os
|
|
||||||
|
|
||||||
import redis
|
import redis
|
||||||
|
from home.src.ta.settings import EnvironmentSettings
|
||||||
|
|
||||||
|
|
||||||
class RedisBase:
|
class RedisBase:
|
||||||
"""connection base for redis"""
|
"""connection base for redis"""
|
||||||
|
|
||||||
REDIS_HOST: str = str(os.environ.get("REDIS_HOST"))
|
NAME_SPACE: str = EnvironmentSettings.REDIS_NAME_SPACE
|
||||||
REDIS_PORT: int = int(os.environ.get("REDIS_PORT") or 6379)
|
|
||||||
NAME_SPACE: str = "ta:"
|
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.conn = redis.Redis(host=self.REDIS_HOST, port=self.REDIS_PORT)
|
self.conn = redis.Redis(
|
||||||
|
host=EnvironmentSettings.REDIS_HOST,
|
||||||
|
port=EnvironmentSettings.REDIS_PORT,
|
||||||
|
decode_responses=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class RedisArchivist(RedisBase):
|
class RedisArchivist(RedisBase):
|
||||||
|
@ -41,6 +43,7 @@ class RedisArchivist(RedisBase):
|
||||||
message: dict,
|
message: dict,
|
||||||
path: str = ".",
|
path: str = ".",
|
||||||
expire: bool | int = False,
|
expire: bool | int = False,
|
||||||
|
save: bool = False,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""write new message to redis"""
|
"""write new message to redis"""
|
||||||
self.conn.execute_command(
|
self.conn.execute_command(
|
||||||
|
@ -54,6 +57,16 @@ class RedisArchivist(RedisBase):
|
||||||
secs = expire
|
secs = expire
|
||||||
self.conn.execute_command("EXPIRE", self.NAME_SPACE + key, secs)
|
self.conn.execute_command("EXPIRE", self.NAME_SPACE + key, secs)
|
||||||
|
|
||||||
|
if save:
|
||||||
|
self.bg_save()
|
||||||
|
|
||||||
|
def bg_save(self) -> None:
|
||||||
|
"""save to aof"""
|
||||||
|
try:
|
||||||
|
self.conn.bgsave()
|
||||||
|
except redis.exceptions.ResponseError:
|
||||||
|
pass
|
||||||
|
|
||||||
def get_message(self, key: str) -> dict:
|
def get_message(self, key: str) -> dict:
|
||||||
"""get message dict from redis"""
|
"""get message dict from redis"""
|
||||||
reply = self.conn.execute_command("JSON.GET", self.NAME_SPACE + key)
|
reply = self.conn.execute_command("JSON.GET", self.NAME_SPACE + key)
|
||||||
|
@ -70,7 +83,7 @@ class RedisArchivist(RedisBase):
|
||||||
if not reply:
|
if not reply:
|
||||||
return []
|
return []
|
||||||
|
|
||||||
return [i.decode().lstrip(self.NAME_SPACE) for i in reply]
|
return [i.lstrip(self.NAME_SPACE) for i in reply]
|
||||||
|
|
||||||
def list_items(self, query: str) -> list:
|
def list_items(self, query: str) -> list:
|
||||||
"""list all matches"""
|
"""list all matches"""
|
||||||
|
@ -87,65 +100,90 @@ class RedisArchivist(RedisBase):
|
||||||
|
|
||||||
|
|
||||||
class RedisQueue(RedisBase):
|
class RedisQueue(RedisBase):
|
||||||
"""dynamically interact with queues in redis"""
|
"""
|
||||||
|
dynamically interact with queues in redis using sorted set
|
||||||
|
- low score number is first in queue
|
||||||
|
- add new items with high score number
|
||||||
|
|
||||||
|
queue names in use:
|
||||||
|
download:channel channels during download
|
||||||
|
download:playlist:full playlists during dl for full refresh
|
||||||
|
download:playlist:quick playlists during dl for quick refresh
|
||||||
|
download:video videos during downloads
|
||||||
|
index:comment videos needing comment indexing
|
||||||
|
reindex:ta_video reindex videos
|
||||||
|
reindex:ta_channel reindex channels
|
||||||
|
reindex:ta_playlist reindex playlists
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(self, queue_name: str):
|
def __init__(self, queue_name: str):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.key = f"{self.NAME_SPACE}{queue_name}"
|
self.key = f"{self.NAME_SPACE}{queue_name}"
|
||||||
|
|
||||||
def get_all(self):
|
def get_all(self) -> list[str]:
|
||||||
"""return all elements in list"""
|
"""return all elements in list"""
|
||||||
result = self.conn.execute_command("LRANGE", self.key, 0, -1)
|
result = self.conn.zrange(self.key, 0, -1)
|
||||||
all_elements = [i.decode() for i in result]
|
return result
|
||||||
return all_elements
|
|
||||||
|
|
||||||
def length(self) -> int:
|
def length(self) -> int:
|
||||||
"""return total elements in list"""
|
"""return total elements in list"""
|
||||||
return self.conn.execute_command("LLEN", self.key)
|
return self.conn.zcard(self.key)
|
||||||
|
|
||||||
def in_queue(self, element) -> str | bool:
|
def in_queue(self, element) -> str | bool:
|
||||||
"""check if element is in list"""
|
"""check if element is in list"""
|
||||||
result = self.conn.execute_command("LPOS", self.key, element)
|
result = self.conn.zrank(self.key, element)
|
||||||
if result is not None:
|
if result is not None:
|
||||||
return "in_queue"
|
return "in_queue"
|
||||||
|
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def add_list(self, to_add):
|
def add(self, to_add: str) -> None:
|
||||||
|
"""add single item to queue"""
|
||||||
|
if not to_add:
|
||||||
|
return
|
||||||
|
|
||||||
|
next_score = self._get_next_score()
|
||||||
|
self.conn.zadd(self.key, {to_add: next_score})
|
||||||
|
|
||||||
|
def add_list(self, to_add: list) -> None:
|
||||||
"""add list to queue"""
|
"""add list to queue"""
|
||||||
self.conn.execute_command("RPUSH", self.key, *to_add)
|
if not to_add:
|
||||||
|
return
|
||||||
|
|
||||||
def add_priority(self, to_add: str) -> None:
|
next_score = self._get_next_score()
|
||||||
"""add single video to front of queue"""
|
mapping = {i[1]: next_score + i[0] for i in enumerate(to_add)}
|
||||||
item: str = json.dumps(to_add)
|
self.conn.zadd(self.key, mapping)
|
||||||
self.clear_item(item)
|
|
||||||
self.conn.execute_command("LPUSH", self.key, item)
|
|
||||||
|
|
||||||
def get_next(self) -> str | bool:
|
def max_score(self) -> int | None:
|
||||||
"""return next element in the queue, False if none"""
|
"""get max score"""
|
||||||
result = self.conn.execute_command("LPOP", self.key)
|
last = self.conn.zrange(self.key, -1, -1, withscores=True)
|
||||||
|
if not last:
|
||||||
|
return None
|
||||||
|
|
||||||
|
return int(last[0][1])
|
||||||
|
|
||||||
|
def _get_next_score(self) -> float:
|
||||||
|
"""get next score in queue to append"""
|
||||||
|
last = self.conn.zrange(self.key, -1, -1, withscores=True)
|
||||||
|
if not last:
|
||||||
|
return 1.0
|
||||||
|
|
||||||
|
return last[0][1] + 1
|
||||||
|
|
||||||
|
def get_next(self) -> tuple[str | None, int | None]:
|
||||||
|
"""return next element in the queue, if available"""
|
||||||
|
result = self.conn.zpopmin(self.key)
|
||||||
if not result:
|
if not result:
|
||||||
return False
|
return None, None
|
||||||
|
|
||||||
next_element = result.decode()
|
item, idx = result[0][0], int(result[0][1])
|
||||||
return next_element
|
|
||||||
|
return item, idx
|
||||||
|
|
||||||
def clear(self) -> None:
|
def clear(self) -> None:
|
||||||
"""delete list from redis"""
|
"""delete list from redis"""
|
||||||
self.conn.execute_command("DEL", self.key)
|
self.conn.delete(self.key)
|
||||||
|
|
||||||
def clear_item(self, to_clear: str) -> None:
|
|
||||||
"""remove single item from list if it's there"""
|
|
||||||
self.conn.execute_command("LREM", self.key, 0, to_clear)
|
|
||||||
|
|
||||||
def trim(self, size: int) -> None:
|
|
||||||
"""trim the queue based on settings amount"""
|
|
||||||
self.conn.execute_command("LTRIM", self.key, 0, size)
|
|
||||||
|
|
||||||
def has_item(self) -> bool:
|
|
||||||
"""check if queue as at least one pending item"""
|
|
||||||
result = self.conn.execute_command("LRANGE", self.key, 0, 0)
|
|
||||||
return bool(result)
|
|
||||||
|
|
||||||
|
|
||||||
class TaskRedis(RedisBase):
|
class TaskRedis(RedisBase):
|
||||||
|
@ -158,7 +196,7 @@ class TaskRedis(RedisBase):
|
||||||
def get_all(self) -> list:
|
def get_all(self) -> list:
|
||||||
"""return all tasks"""
|
"""return all tasks"""
|
||||||
all_keys = self.conn.execute_command("KEYS", f"{self.BASE}*")
|
all_keys = self.conn.execute_command("KEYS", f"{self.BASE}*")
|
||||||
return [i.decode().replace(self.BASE, "") for i in all_keys]
|
return [i.replace(self.BASE, "") for i in all_keys]
|
||||||
|
|
||||||
def get_single(self, task_id: str) -> dict:
|
def get_single(self, task_id: str) -> dict:
|
||||||
"""return content of single task"""
|
"""return content of single task"""
|
||||||
|
@ -166,7 +204,7 @@ class TaskRedis(RedisBase):
|
||||||
if not result:
|
if not result:
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
return json.loads(result.decode())
|
return json.loads(result)
|
||||||
|
|
||||||
def set_key(
|
def set_key(
|
||||||
self, task_id: str, message: dict, expire: bool | int = False
|
self, task_id: str, message: dict, expire: bool | int = False
|
||||||
|
|
|
@ -0,0 +1,125 @@
|
||||||
|
"""
|
||||||
|
Functionality:
|
||||||
|
- Static Task config values
|
||||||
|
- Type definitions
|
||||||
|
- separate to avoid circular imports
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import TypedDict
|
||||||
|
|
||||||
|
|
||||||
|
class TaskItemConfig(TypedDict):
|
||||||
|
"""describes a task item config"""
|
||||||
|
|
||||||
|
title: str
|
||||||
|
group: str
|
||||||
|
api_start: bool
|
||||||
|
api_stop: bool
|
||||||
|
|
||||||
|
|
||||||
|
UPDATE_SUBSCRIBED: TaskItemConfig = {
|
||||||
|
"title": "Rescan your Subscriptions",
|
||||||
|
"group": "download:scan",
|
||||||
|
"api_start": True,
|
||||||
|
"api_stop": True,
|
||||||
|
}
|
||||||
|
|
||||||
|
DOWNLOAD_PENDING: TaskItemConfig = {
|
||||||
|
"title": "Downloading",
|
||||||
|
"group": "download:run",
|
||||||
|
"api_start": True,
|
||||||
|
"api_stop": True,
|
||||||
|
}
|
||||||
|
|
||||||
|
EXTRACT_DOWNLOAD: TaskItemConfig = {
|
||||||
|
"title": "Add to download queue",
|
||||||
|
"group": "download:add",
|
||||||
|
"api_start": False,
|
||||||
|
"api_stop": True,
|
||||||
|
}
|
||||||
|
|
||||||
|
CHECK_REINDEX: TaskItemConfig = {
|
||||||
|
"title": "Reindex Documents",
|
||||||
|
"group": "reindex:run",
|
||||||
|
"api_start": False,
|
||||||
|
"api_stop": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
MANUAL_IMPORT: TaskItemConfig = {
|
||||||
|
"title": "Manual video import",
|
||||||
|
"group": "setting:import",
|
||||||
|
"api_start": True,
|
||||||
|
"api_stop": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
RUN_BACKUP: TaskItemConfig = {
|
||||||
|
"title": "Index Backup",
|
||||||
|
"group": "setting:backup",
|
||||||
|
"api_start": True,
|
||||||
|
"api_stop": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
RESTORE_BACKUP: TaskItemConfig = {
|
||||||
|
"title": "Restore Backup",
|
||||||
|
"group": "setting:restore",
|
||||||
|
"api_start": False,
|
||||||
|
"api_stop": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
RESCAN_FILESYSTEM: TaskItemConfig = {
|
||||||
|
"title": "Rescan your Filesystem",
|
||||||
|
"group": "setting:filesystemscan",
|
||||||
|
"api_start": True,
|
||||||
|
"api_stop": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
THUMBNAIL_CHECK: TaskItemConfig = {
|
||||||
|
"title": "Check your Thumbnails",
|
||||||
|
"group": "setting:thumbnailcheck",
|
||||||
|
"api_start": True,
|
||||||
|
"api_stop": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
RESYNC_THUMBS: TaskItemConfig = {
|
||||||
|
"title": "Sync Thumbnails to Media Files",
|
||||||
|
"group": "setting:thumbnailsync",
|
||||||
|
"api_start": True,
|
||||||
|
"api_stop": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
INDEX_PLAYLISTS: TaskItemConfig = {
|
||||||
|
"title": "Index Channel Playlist",
|
||||||
|
"group": "channel:indexplaylist",
|
||||||
|
"api_start": False,
|
||||||
|
"api_stop": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
SUBSCRIBE_TO: TaskItemConfig = {
|
||||||
|
"title": "Add Subscription",
|
||||||
|
"group": "subscription:add",
|
||||||
|
"api_start": False,
|
||||||
|
"api_stop": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
VERSION_CHECK: TaskItemConfig = {
|
||||||
|
"title": "Look for new Version",
|
||||||
|
"group": "",
|
||||||
|
"api_start": False,
|
||||||
|
"api_stop": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
TASK_CONFIG: dict[str, TaskItemConfig] = {
|
||||||
|
"update_subscribed": UPDATE_SUBSCRIBED,
|
||||||
|
"download_pending": DOWNLOAD_PENDING,
|
||||||
|
"extract_download": EXTRACT_DOWNLOAD,
|
||||||
|
"check_reindex": CHECK_REINDEX,
|
||||||
|
"manual_import": MANUAL_IMPORT,
|
||||||
|
"run_backup": RUN_BACKUP,
|
||||||
|
"restore_backup": RESTORE_BACKUP,
|
||||||
|
"rescan_filesystem": RESCAN_FILESYSTEM,
|
||||||
|
"thumbnail_check": THUMBNAIL_CHECK,
|
||||||
|
"resync_thumbs": RESYNC_THUMBS,
|
||||||
|
"index_playlists": INDEX_PLAYLISTS,
|
||||||
|
"subscribe_to": SUBSCRIBE_TO,
|
||||||
|
"version_check": VERSION_CHECK,
|
||||||
|
}
|
|
@ -4,8 +4,9 @@ functionality:
|
||||||
- handle threads and locks
|
- handle threads and locks
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from home import tasks as ta_tasks
|
from home.celery import app as celery_app
|
||||||
from home.src.ta.ta_redis import RedisArchivist, TaskRedis
|
from home.src.ta.ta_redis import RedisArchivist, TaskRedis
|
||||||
|
from home.src.ta.task_config import TASK_CONFIG
|
||||||
|
|
||||||
|
|
||||||
class TaskManager:
|
class TaskManager:
|
||||||
|
@ -86,7 +87,7 @@ class TaskCommand:
|
||||||
|
|
||||||
def start(self, task_name):
|
def start(self, task_name):
|
||||||
"""start task by task_name, only pass task that don't take args"""
|
"""start task by task_name, only pass task that don't take args"""
|
||||||
task = ta_tasks.app.tasks.get(task_name).delay()
|
task = celery_app.tasks.get(task_name).delay()
|
||||||
message = {
|
message = {
|
||||||
"task_id": task.id,
|
"task_id": task.id,
|
||||||
"status": task.status,
|
"status": task.status,
|
||||||
|
@ -104,7 +105,7 @@ class TaskCommand:
|
||||||
handler = TaskRedis()
|
handler = TaskRedis()
|
||||||
|
|
||||||
task = handler.get_single(task_id)
|
task = handler.get_single(task_id)
|
||||||
if not task["name"] in ta_tasks.BaseTask.TASK_CONFIG:
|
if not task["name"] in TASK_CONFIG:
|
||||||
raise ValueError
|
raise ValueError
|
||||||
|
|
||||||
handler.set_command(task_id, "STOP")
|
handler.set_command(task_id, "STOP")
|
||||||
|
@ -113,4 +114,4 @@ class TaskCommand:
|
||||||
def kill(self, task_id):
|
def kill(self, task_id):
|
||||||
"""send kill signal to task_id"""
|
"""send kill signal to task_id"""
|
||||||
print(f"[task][{task_id}]: received KILL signal.")
|
print(f"[task][{task_id}]: received KILL signal.")
|
||||||
ta_tasks.app.control.revoke(task_id, terminate=True)
|
celery_app.control.revoke(task_id, terminate=True)
|
||||||
|
|
|
@ -92,7 +92,7 @@ class Parser:
|
||||||
item_type = "video"
|
item_type = "video"
|
||||||
elif len_id_str == 24:
|
elif len_id_str == 24:
|
||||||
item_type = "channel"
|
item_type = "channel"
|
||||||
elif len_id_str in (34, 18):
|
elif len_id_str in (34, 26, 18) or id_str.startswith("TA_playlist_"):
|
||||||
item_type = "playlist"
|
item_type = "playlist"
|
||||||
else:
|
else:
|
||||||
raise ValueError(f"not a valid id_str: {id_str}")
|
raise ValueError(f"not a valid id_str: {id_str}")
|
||||||
|
|
|
@ -0,0 +1,142 @@
|
||||||
|
"""
|
||||||
|
Functionality:
|
||||||
|
- read and write user config backed by ES
|
||||||
|
- encapsulate persistence of user properties
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import TypedDict
|
||||||
|
|
||||||
|
from home.src.es.connect import ElasticWrap
|
||||||
|
from home.src.ta.helper import get_stylesheets
|
||||||
|
|
||||||
|
|
||||||
|
class UserConfigType(TypedDict, total=False):
|
||||||
|
"""describes the user configuration"""
|
||||||
|
|
||||||
|
stylesheet: str
|
||||||
|
page_size: int
|
||||||
|
sort_by: str
|
||||||
|
sort_order: str
|
||||||
|
view_style_home: str
|
||||||
|
view_style_channel: str
|
||||||
|
view_style_downloads: str
|
||||||
|
view_style_playlist: str
|
||||||
|
grid_items: int
|
||||||
|
hide_watched: bool
|
||||||
|
show_ignored_only: bool
|
||||||
|
show_subed_only: bool
|
||||||
|
sponsorblock_id: str
|
||||||
|
|
||||||
|
|
||||||
|
class UserConfig:
|
||||||
|
"""Handle settings for an individual user"""
|
||||||
|
|
||||||
|
_DEFAULT_USER_SETTINGS = UserConfigType(
|
||||||
|
stylesheet="dark.css",
|
||||||
|
page_size=12,
|
||||||
|
sort_by="published",
|
||||||
|
sort_order="desc",
|
||||||
|
view_style_home="grid",
|
||||||
|
view_style_channel="list",
|
||||||
|
view_style_downloads="list",
|
||||||
|
view_style_playlist="grid",
|
||||||
|
grid_items=3,
|
||||||
|
hide_watched=False,
|
||||||
|
show_ignored_only=False,
|
||||||
|
show_subed_only=False,
|
||||||
|
sponsorblock_id=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
VALID_STYLESHEETS = get_stylesheets()
|
||||||
|
VALID_VIEW_STYLE = ["grid", "list"]
|
||||||
|
VALID_SORT_ORDER = ["asc", "desc"]
|
||||||
|
VALID_SORT_BY = [
|
||||||
|
"published",
|
||||||
|
"downloaded",
|
||||||
|
"views",
|
||||||
|
"likes",
|
||||||
|
"duration",
|
||||||
|
"filesize",
|
||||||
|
]
|
||||||
|
VALID_GRID_ITEMS = range(3, 8)
|
||||||
|
|
||||||
|
def __init__(self, user_id: str):
|
||||||
|
self._user_id: str = user_id
|
||||||
|
self._config: UserConfigType = self.get_config()
|
||||||
|
|
||||||
|
def get_value(self, key: str):
|
||||||
|
"""Get the given key from the users configuration
|
||||||
|
|
||||||
|
Throws a KeyError if the requested Key is not a permitted value"""
|
||||||
|
if key not in self._DEFAULT_USER_SETTINGS:
|
||||||
|
raise KeyError(f"Unable to read config for unknown key '{key}'")
|
||||||
|
|
||||||
|
return self._config.get(key) or self._DEFAULT_USER_SETTINGS.get(key)
|
||||||
|
|
||||||
|
def set_value(self, key: str, value: str | bool | int):
|
||||||
|
"""Set or replace a configuration value for the user"""
|
||||||
|
self._validate(key, value)
|
||||||
|
old = self.get_value(key)
|
||||||
|
self._config[key] = value
|
||||||
|
|
||||||
|
# Upsert this property (creating a record if not exists)
|
||||||
|
es_payload = {"doc": {"config": {key: value}}, "doc_as_upsert": True}
|
||||||
|
es_document_path = f"ta_config/_update/user_{self._user_id}"
|
||||||
|
response, status = ElasticWrap(es_document_path).post(es_payload)
|
||||||
|
if status < 200 or status > 299:
|
||||||
|
raise ValueError(f"Failed storing user value {status}: {response}")
|
||||||
|
|
||||||
|
print(f"User {self._user_id} value '{key}' change: {old} -> {value}")
|
||||||
|
|
||||||
|
def _validate(self, key, value):
|
||||||
|
"""validate key and value"""
|
||||||
|
if not self._user_id:
|
||||||
|
raise ValueError("Unable to persist config for null user_id")
|
||||||
|
|
||||||
|
if key not in self._DEFAULT_USER_SETTINGS:
|
||||||
|
raise KeyError(
|
||||||
|
f"Unable to persist config for an unknown key '{key}'"
|
||||||
|
)
|
||||||
|
|
||||||
|
valid_values = {
|
||||||
|
"stylesheet": self.VALID_STYLESHEETS,
|
||||||
|
"sort_by": self.VALID_SORT_BY,
|
||||||
|
"sort_order": self.VALID_SORT_ORDER,
|
||||||
|
"view_style_home": self.VALID_VIEW_STYLE,
|
||||||
|
"view_style_channel": self.VALID_VIEW_STYLE,
|
||||||
|
"view_style_download": self.VALID_VIEW_STYLE,
|
||||||
|
"view_style_playlist": self.VALID_VIEW_STYLE,
|
||||||
|
"grid_items": self.VALID_GRID_ITEMS,
|
||||||
|
"page_size": int,
|
||||||
|
"hide_watched": bool,
|
||||||
|
"show_ignored_only": bool,
|
||||||
|
"show_subed_only": bool,
|
||||||
|
}
|
||||||
|
validation_value = valid_values.get(key)
|
||||||
|
|
||||||
|
if isinstance(validation_value, (list, range)):
|
||||||
|
if value not in validation_value:
|
||||||
|
raise ValueError(f"Invalid value for {key}: {value}")
|
||||||
|
elif validation_value == int:
|
||||||
|
if not isinstance(value, int):
|
||||||
|
raise ValueError(f"Invalid value for {key}: {value}")
|
||||||
|
elif validation_value == bool:
|
||||||
|
if not isinstance(value, bool):
|
||||||
|
raise ValueError(f"Invalid value for {key}: {value}")
|
||||||
|
|
||||||
|
def get_config(self) -> UserConfigType:
|
||||||
|
"""get config from ES or load from the application defaults"""
|
||||||
|
if not self._user_id:
|
||||||
|
# this is for a non logged-in user so use all the defaults
|
||||||
|
return {}
|
||||||
|
|
||||||
|
# Does this user have configuration stored in ES
|
||||||
|
es_document_path = f"ta_config/_doc/user_{self._user_id}"
|
||||||
|
response, status = ElasticWrap(es_document_path).get(print_error=False)
|
||||||
|
if status == 200 and "_source" in response.keys():
|
||||||
|
source = response.get("_source")
|
||||||
|
if "config" in source.keys():
|
||||||
|
return source.get("config")
|
||||||
|
|
||||||
|
# There is no config in ES
|
||||||
|
return {}
|
|
@ -1,14 +1,12 @@
|
||||||
"""
|
"""
|
||||||
Functionality:
|
Functionality:
|
||||||
- initiate celery app
|
|
||||||
- collect tasks
|
- collect tasks
|
||||||
- user config changes won't get applied here
|
- handle task callbacks
|
||||||
because tasks are initiated at application start
|
- handle task notifications
|
||||||
|
- handle task locking
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import os
|
from celery import Task, shared_task
|
||||||
|
|
||||||
from celery import Celery, Task, shared_task
|
|
||||||
from home.src.download.queue import PendingList
|
from home.src.download.queue import PendingList
|
||||||
from home.src.download.subscriptions import (
|
from home.src.download.subscriptions import (
|
||||||
SubscriptionHandler,
|
SubscriptionHandler,
|
||||||
|
@ -19,27 +17,15 @@ from home.src.download.yt_dlp_handler import VideoDownloader
|
||||||
from home.src.es.backup import ElasticBackup
|
from home.src.es.backup import ElasticBackup
|
||||||
from home.src.es.index_setup import ElasitIndexWrap
|
from home.src.es.index_setup import ElasitIndexWrap
|
||||||
from home.src.index.channel import YoutubeChannel
|
from home.src.index.channel import YoutubeChannel
|
||||||
from home.src.index.filesystem import Filesystem
|
from home.src.index.filesystem import Scanner
|
||||||
from home.src.index.manual import ImportFolderScanner
|
from home.src.index.manual import ImportFolderScanner
|
||||||
from home.src.index.reindex import Reindex, ReindexManual, ReindexPopulate
|
from home.src.index.reindex import Reindex, ReindexManual, ReindexPopulate
|
||||||
from home.src.ta.config import AppConfig, ReleaseVersion, ScheduleBuilder
|
from home.src.ta.config import ReleaseVersion
|
||||||
|
from home.src.ta.notify import Notifications
|
||||||
from home.src.ta.ta_redis import RedisArchivist
|
from home.src.ta.ta_redis import RedisArchivist
|
||||||
|
from home.src.ta.task_config import TASK_CONFIG
|
||||||
from home.src.ta.task_manager import TaskManager
|
from home.src.ta.task_manager import TaskManager
|
||||||
|
from home.src.ta.urlparser import Parser
|
||||||
CONFIG = AppConfig().config
|
|
||||||
REDIS_HOST = os.environ.get("REDIS_HOST")
|
|
||||||
REDIS_PORT = os.environ.get("REDIS_PORT") or 6379
|
|
||||||
|
|
||||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
|
|
||||||
app = Celery(
|
|
||||||
"tasks",
|
|
||||||
broker=f"redis://{REDIS_HOST}:{REDIS_PORT}",
|
|
||||||
backend=f"redis://{REDIS_HOST}:{REDIS_PORT}",
|
|
||||||
result_extended=True,
|
|
||||||
)
|
|
||||||
app.config_from_object("django.conf:settings", namespace="ta:")
|
|
||||||
app.autodiscover_tasks()
|
|
||||||
app.conf.timezone = os.environ.get("TZ") or "UTC"
|
|
||||||
|
|
||||||
|
|
||||||
class BaseTask(Task):
|
class BaseTask(Task):
|
||||||
|
@ -47,72 +33,11 @@ class BaseTask(Task):
|
||||||
|
|
||||||
# pylint: disable=abstract-method
|
# pylint: disable=abstract-method
|
||||||
|
|
||||||
TASK_CONFIG = {
|
|
||||||
"update_subscribed": {
|
|
||||||
"title": "Rescan your Subscriptions",
|
|
||||||
"group": "download:scan",
|
|
||||||
"api-start": True,
|
|
||||||
"api-stop": True,
|
|
||||||
},
|
|
||||||
"download_pending": {
|
|
||||||
"title": "Downloading",
|
|
||||||
"group": "download:run",
|
|
||||||
"api-start": True,
|
|
||||||
"api-stop": True,
|
|
||||||
},
|
|
||||||
"extract_download": {
|
|
||||||
"title": "Add to download queue",
|
|
||||||
"group": "download:add",
|
|
||||||
"api-stop": True,
|
|
||||||
},
|
|
||||||
"check_reindex": {
|
|
||||||
"title": "Reindex Documents",
|
|
||||||
"group": "reindex:run",
|
|
||||||
},
|
|
||||||
"manual_import": {
|
|
||||||
"title": "Manual video import",
|
|
||||||
"group": "setting:import",
|
|
||||||
"api-start": True,
|
|
||||||
},
|
|
||||||
"run_backup": {
|
|
||||||
"title": "Index Backup",
|
|
||||||
"group": "setting:backup",
|
|
||||||
"api-start": True,
|
|
||||||
},
|
|
||||||
"restore_backup": {
|
|
||||||
"title": "Restore Backup",
|
|
||||||
"group": "setting:restore",
|
|
||||||
},
|
|
||||||
"rescan_filesystem": {
|
|
||||||
"title": "Rescan your Filesystem",
|
|
||||||
"group": "setting:filesystemscan",
|
|
||||||
"api-start": True,
|
|
||||||
},
|
|
||||||
"thumbnail_check": {
|
|
||||||
"title": "Check your Thumbnails",
|
|
||||||
"group": "setting:thumbnailcheck",
|
|
||||||
"api-start": True,
|
|
||||||
},
|
|
||||||
"resync_thumbs": {
|
|
||||||
"title": "Sync Thumbnails to Media Files",
|
|
||||||
"group": "setting:thumbnailsync",
|
|
||||||
"api-start": True,
|
|
||||||
},
|
|
||||||
"index_playlists": {
|
|
||||||
"title": "Index Channel Playlist",
|
|
||||||
"group": "channel:indexplaylist",
|
|
||||||
},
|
|
||||||
"subscribe_to": {
|
|
||||||
"title": "Add Subscription",
|
|
||||||
"group": "subscription:add",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
def on_failure(self, exc, task_id, args, kwargs, einfo):
|
def on_failure(self, exc, task_id, args, kwargs, einfo):
|
||||||
"""callback for task failure"""
|
"""callback for task failure"""
|
||||||
print(f"{task_id} Failed callback")
|
print(f"{task_id} Failed callback")
|
||||||
message, key = self._build_message(level="error")
|
message, key = self._build_message(level="error")
|
||||||
message.update({"messages": ["Task failed"]})
|
message.update({"messages": [f"Task failed: {exc}"]})
|
||||||
RedisArchivist().set_message(key, message, expire=20)
|
RedisArchivist().set_message(key, message, expire=20)
|
||||||
|
|
||||||
def on_success(self, retval, task_id, args, kwargs):
|
def on_success(self, retval, task_id, args, kwargs):
|
||||||
|
@ -129,6 +54,12 @@ class BaseTask(Task):
|
||||||
message.update({"messages": ["New task received."]})
|
message.update({"messages": ["New task received."]})
|
||||||
RedisArchivist().set_message(key, message)
|
RedisArchivist().set_message(key, message)
|
||||||
|
|
||||||
|
def after_return(self, status, retval, task_id, args, kwargs, einfo):
|
||||||
|
"""callback after task returns"""
|
||||||
|
print(f"{task_id} return callback")
|
||||||
|
task_title = TASK_CONFIG.get(self.name).get("title")
|
||||||
|
Notifications(self.name).send(task_id, task_title)
|
||||||
|
|
||||||
def send_progress(self, message_lines, progress=False, title=False):
|
def send_progress(self, message_lines, progress=False, title=False):
|
||||||
"""send progress message"""
|
"""send progress message"""
|
||||||
message, key = self._build_message()
|
message, key = self._build_message()
|
||||||
|
@ -146,7 +77,7 @@ class BaseTask(Task):
|
||||||
def _build_message(self, level="info"):
|
def _build_message(self, level="info"):
|
||||||
"""build message dict"""
|
"""build message dict"""
|
||||||
task_id = self.request.id
|
task_id = self.request.id
|
||||||
message = self.TASK_CONFIG.get(self.name).copy()
|
message = TASK_CONFIG.get(self.name).copy()
|
||||||
message.update({"level": level, "id": task_id})
|
message.update({"level": level, "id": task_id})
|
||||||
task_result = TaskManager().get_task(task_id)
|
task_result = TaskManager().get_task(task_id)
|
||||||
if task_result:
|
if task_result:
|
||||||
|
@ -168,38 +99,62 @@ def update_subscribed(self):
|
||||||
if manager.is_pending(self):
|
if manager.is_pending(self):
|
||||||
print(f"[task][{self.name}] rescan already running")
|
print(f"[task][{self.name}] rescan already running")
|
||||||
self.send_progress("Rescan already in progress.")
|
self.send_progress("Rescan already in progress.")
|
||||||
return
|
return None
|
||||||
|
|
||||||
manager.init(self)
|
manager.init(self)
|
||||||
missing_videos = SubscriptionScanner(task=self).scan()
|
handler = SubscriptionScanner(task=self)
|
||||||
|
missing_videos = handler.scan()
|
||||||
|
auto_start = handler.auto_start
|
||||||
if missing_videos:
|
if missing_videos:
|
||||||
print(missing_videos)
|
print(missing_videos)
|
||||||
extrac_dl.delay(missing_videos)
|
extrac_dl.delay(missing_videos, auto_start=auto_start)
|
||||||
|
message = f"Found {len(missing_videos)} videos to add to the queue."
|
||||||
|
return message
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
@shared_task(name="download_pending", bind=True, base=BaseTask)
|
@shared_task(name="download_pending", bind=True, base=BaseTask)
|
||||||
def download_pending(self, from_queue=True):
|
def download_pending(self, auto_only=False):
|
||||||
"""download latest pending videos"""
|
"""download latest pending videos"""
|
||||||
manager = TaskManager()
|
manager = TaskManager()
|
||||||
if manager.is_pending(self):
|
if manager.is_pending(self):
|
||||||
print(f"[task][{self.name}] download queue already running")
|
print(f"[task][{self.name}] download queue already running")
|
||||||
self.send_progress("Download Queue is already running.")
|
self.send_progress("Download Queue is already running.")
|
||||||
return
|
return None
|
||||||
|
|
||||||
manager.init(self)
|
manager.init(self)
|
||||||
downloader = VideoDownloader(task=self)
|
downloader = VideoDownloader(task=self)
|
||||||
if from_queue:
|
videos_downloaded = downloader.run_queue(auto_only=auto_only)
|
||||||
downloader.add_pending()
|
|
||||||
downloader.run_queue()
|
if videos_downloaded:
|
||||||
|
return f"downloaded {videos_downloaded} video(s)."
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
@shared_task(name="extract_download", bind=True, base=BaseTask)
|
@shared_task(name="extract_download", bind=True, base=BaseTask)
|
||||||
def extrac_dl(self, youtube_ids):
|
def extrac_dl(self, youtube_ids, auto_start=False, status="pending"):
|
||||||
"""parse list passed and add to pending"""
|
"""parse list passed and add to pending"""
|
||||||
TaskManager().init(self)
|
TaskManager().init(self)
|
||||||
pending_handler = PendingList(youtube_ids=youtube_ids, task=self)
|
if isinstance(youtube_ids, str):
|
||||||
|
to_add = Parser(youtube_ids).parse()
|
||||||
|
else:
|
||||||
|
to_add = youtube_ids
|
||||||
|
|
||||||
|
pending_handler = PendingList(youtube_ids=to_add, task=self)
|
||||||
pending_handler.parse_url_list()
|
pending_handler.parse_url_list()
|
||||||
pending_handler.add_to_pending()
|
videos_added = pending_handler.add_to_pending(
|
||||||
|
status=status, auto_start=auto_start
|
||||||
|
)
|
||||||
|
|
||||||
|
if auto_start:
|
||||||
|
download_pending.delay(auto_only=True)
|
||||||
|
|
||||||
|
if videos_added:
|
||||||
|
return f"added {len(videos_added)} Videos to Queue"
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
@shared_task(bind=True, name="check_reindex", base=BaseTask)
|
@shared_task(bind=True, name="check_reindex", base=BaseTask)
|
||||||
|
@ -223,11 +178,15 @@ def check_reindex(self, data=False, extract_videos=False):
|
||||||
populate = ReindexPopulate()
|
populate = ReindexPopulate()
|
||||||
print(f"[task][{self.name}] reindex outdated documents")
|
print(f"[task][{self.name}] reindex outdated documents")
|
||||||
self.send_progress("Add recent documents to the reindex Queue.")
|
self.send_progress("Add recent documents to the reindex Queue.")
|
||||||
|
populate.get_interval()
|
||||||
populate.add_recent()
|
populate.add_recent()
|
||||||
self.send_progress("Add outdated documents to the reindex Queue.")
|
self.send_progress("Add outdated documents to the reindex Queue.")
|
||||||
populate.add_outdated()
|
populate.add_outdated()
|
||||||
|
|
||||||
Reindex(task=self).reindex_all()
|
handler = Reindex(task=self)
|
||||||
|
handler.reindex_all()
|
||||||
|
|
||||||
|
return handler.build_message()
|
||||||
|
|
||||||
|
|
||||||
@shared_task(bind=True, name="manual_import", base=BaseTask)
|
@shared_task(bind=True, name="manual_import", base=BaseTask)
|
||||||
|
@ -263,7 +222,7 @@ def run_restore_backup(self, filename):
|
||||||
if manager.is_pending(self):
|
if manager.is_pending(self):
|
||||||
print(f"[task][{self.name}] restore is already running")
|
print(f"[task][{self.name}] restore is already running")
|
||||||
self.send_progress("Restore is already running.")
|
self.send_progress("Restore is already running.")
|
||||||
return
|
return None
|
||||||
|
|
||||||
manager.init(self)
|
manager.init(self)
|
||||||
self.send_progress(["Reset your Index"])
|
self.send_progress(["Reset your Index"])
|
||||||
|
@ -271,6 +230,8 @@ def run_restore_backup(self, filename):
|
||||||
ElasticBackup(task=self).restore(filename)
|
ElasticBackup(task=self).restore(filename)
|
||||||
print("index restore finished")
|
print("index restore finished")
|
||||||
|
|
||||||
|
return f"backup restore completed: {filename}"
|
||||||
|
|
||||||
|
|
||||||
@shared_task(bind=True, name="rescan_filesystem", base=BaseTask)
|
@shared_task(bind=True, name="rescan_filesystem", base=BaseTask)
|
||||||
def rescan_filesystem(self):
|
def rescan_filesystem(self):
|
||||||
|
@ -282,7 +243,9 @@ def rescan_filesystem(self):
|
||||||
return
|
return
|
||||||
|
|
||||||
manager.init(self)
|
manager.init(self)
|
||||||
Filesystem(task=self).process()
|
handler = Scanner(task=self)
|
||||||
|
handler.scan()
|
||||||
|
handler.apply()
|
||||||
ThumbValidator(task=self).validate()
|
ThumbValidator(task=self).validate()
|
||||||
|
|
||||||
|
|
||||||
|
@ -296,7 +259,9 @@ def thumbnail_check(self):
|
||||||
return
|
return
|
||||||
|
|
||||||
manager.init(self)
|
manager.init(self)
|
||||||
ThumbValidator(task=self).validate()
|
thumnail = ThumbValidator(task=self)
|
||||||
|
thumnail.validate()
|
||||||
|
thumnail.clean_up()
|
||||||
|
|
||||||
|
|
||||||
@shared_task(bind=True, name="resync_thumbs", base=BaseTask)
|
@shared_task(bind=True, name="resync_thumbs", base=BaseTask)
|
||||||
|
@ -313,9 +278,12 @@ def re_sync_thumbs(self):
|
||||||
|
|
||||||
|
|
||||||
@shared_task(bind=True, name="subscribe_to", base=BaseTask)
|
@shared_task(bind=True, name="subscribe_to", base=BaseTask)
|
||||||
def subscribe_to(self, url_str):
|
def subscribe_to(self, url_str: str, expected_type: str | bool = False):
|
||||||
"""take a list of urls to subscribe to"""
|
"""
|
||||||
SubscriptionHandler(url_str, task=self).subscribe()
|
take a list of urls to subscribe to
|
||||||
|
optionally validate expected_type channel / playlist
|
||||||
|
"""
|
||||||
|
SubscriptionHandler(url_str, task=self).subscribe(expected_type)
|
||||||
|
|
||||||
|
|
||||||
@shared_task(bind=True, name="index_playlists", base=BaseTask)
|
@shared_task(bind=True, name="index_playlists", base=BaseTask)
|
||||||
|
@ -329,7 +297,3 @@ def index_channel_playlists(self, channel_id):
|
||||||
def version_check():
|
def version_check():
|
||||||
"""check for new updates"""
|
"""check for new updates"""
|
||||||
ReleaseVersion().check()
|
ReleaseVersion().check()
|
||||||
|
|
||||||
|
|
||||||
# start schedule here
|
|
||||||
app.conf.beat_schedule = ScheduleBuilder().build_schedule()
|
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
{% load static %}
|
{% load static %}
|
||||||
|
{% load auth_extras %}
|
||||||
<!DOCTYPE html>
|
<!DOCTYPE html>
|
||||||
<html lang="en">
|
<html lang="en">
|
||||||
<head>
|
<head>
|
||||||
|
@ -22,11 +23,7 @@
|
||||||
{% else %}
|
{% else %}
|
||||||
<title>TubeArchivist</title>
|
<title>TubeArchivist</title>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
{% if colors == "dark" %}
|
<link rel="stylesheet" href="{% static 'css/' %}{{ stylesheet }}">
|
||||||
<link rel="stylesheet" href="{% static 'css/dark.css' %}">
|
|
||||||
{% else %}
|
|
||||||
<link rel="stylesheet" href="{% static 'css/light.css' %}">
|
|
||||||
{% endif %}
|
|
||||||
<script type="text/javascript" src="{% static 'script.js' %}"></script>
|
<script type="text/javascript" src="{% static 'script.js' %}"></script>
|
||||||
{% if cast %}
|
{% if cast %}
|
||||||
<script type="text/javascript" src="https://www.gstatic.com/cv/js/sender/v1/cast_sender.js?loadCastFramework=1"></script>
|
<script type="text/javascript" src="https://www.gstatic.com/cv/js/sender/v1/cast_sender.js?loadCastFramework=1"></script>
|
||||||
|
@ -36,16 +33,9 @@
|
||||||
<body>
|
<body>
|
||||||
<div class="main-content">
|
<div class="main-content">
|
||||||
<div class="boxed-content">
|
<div class="boxed-content">
|
||||||
<div class="top-banner">
|
<a href="{% url 'home' %}">
|
||||||
<a href="{% url 'home' %}">
|
<div class="top-banner"></div>
|
||||||
{% if colors == 'dark' %}
|
</a>
|
||||||
<img src="{% static 'img/banner-tube-archivist-dark.png' %}" alt="tube-archivist-banner">
|
|
||||||
{% endif %}
|
|
||||||
{% if colors == 'light' %}
|
|
||||||
<img src="{% static 'img/banner-tube-archivist-light.png' %}" alt="tube-archivist-banner">
|
|
||||||
{% endif %}
|
|
||||||
</a>
|
|
||||||
</div>
|
|
||||||
<div class="top-nav">
|
<div class="top-nav">
|
||||||
<div class="nav-items">
|
<div class="nav-items">
|
||||||
<a href="{% url 'home' %}">
|
<a href="{% url 'home' %}">
|
||||||
|
@ -57,9 +47,11 @@
|
||||||
<a href="{% url 'playlist' %}">
|
<a href="{% url 'playlist' %}">
|
||||||
<div class="nav-item">playlists</div>
|
<div class="nav-item">playlists</div>
|
||||||
</a>
|
</a>
|
||||||
|
{% if request.user|has_group:"admin" or request.user.is_staff %}
|
||||||
<a href="{% url 'downloads' %}">
|
<a href="{% url 'downloads' %}">
|
||||||
<div class="nav-item">downloads</div>
|
<div class="nav-item">downloads</div>
|
||||||
</a>
|
</a>
|
||||||
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
<div class="nav-icons">
|
<div class="nav-icons">
|
||||||
<a href="{% url 'search' %}">
|
<a href="{% url 'search' %}">
|
||||||
|
|
|
@ -0,0 +1,20 @@
|
||||||
|
{# Base file for all of the settings pages to ensure a common menu #}
|
||||||
|
{% extends "home/base.html" %}
|
||||||
|
{% load static %}
|
||||||
|
{% load auth_extras %}
|
||||||
|
{% block content %}
|
||||||
|
<div class="boxed-content">
|
||||||
|
<div class="info-box-item child-page-nav">
|
||||||
|
<a href="{% url 'settings' %}"><h3>Dashboard</h3></a>
|
||||||
|
<a href="{% url 'settings_user' %}"><h3>User</h3></a>
|
||||||
|
{% if request.user|has_group:"admin" or request.user.is_staff %}
|
||||||
|
<a href="{% url 'settings_application' %}"><h3>Application</h3></a>
|
||||||
|
<a href="{% url 'settings_scheduling' %}"><h3>Scheduling</h3></a>
|
||||||
|
<a href="{% url 'settings_actions' %}"><h3>Actions</h3></a>
|
||||||
|
{% endif %}
|
||||||
|
</div>
|
||||||
|
<div id="notifications" data=""></div>
|
||||||
|
{% block settings_content %}{% endblock %}
|
||||||
|
</div>
|
||||||
|
<script type="text/javascript" src="{% static 'progress.js' %}"></script>
|
||||||
|
{% endblock content %}
|
|
@ -2,11 +2,13 @@
|
||||||
{% load static %}
|
{% load static %}
|
||||||
{% load humanize %}
|
{% load humanize %}
|
||||||
{% block content %}
|
{% block content %}
|
||||||
|
{% load auth_extras %}
|
||||||
<div class="boxed-content">
|
<div class="boxed-content">
|
||||||
<div class="title-split">
|
<div class="title-split">
|
||||||
<div class="title-bar">
|
<div class="title-bar">
|
||||||
<h1>Channels</h1>
|
<h1>Channels</h1>
|
||||||
</div>
|
</div>
|
||||||
|
{% if request.user|has_group:"admin" or request.user.is_staff %}
|
||||||
<div class="title-split-form">
|
<div class="title-split-form">
|
||||||
<img id="animate-icon" onclick="showForm()" src="{% static 'img/icon-add.svg' %}" alt="add-icon" title="Subscribe to Channels">
|
<img id="animate-icon" onclick="showForm()" src="{% static 'img/icon-add.svg' %}" alt="add-icon" title="Subscribe to Channels">
|
||||||
<div class="show-form">
|
<div class="show-form">
|
||||||
|
@ -17,6 +19,7 @@
|
||||||
</form>
|
</form>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
<div id="notifications" data="subscription"></div>
|
<div id="notifications" data="subscription"></div>
|
||||||
<div class="view-controls">
|
<div class="view-controls">
|
||||||
|
@ -42,33 +45,33 @@
|
||||||
{% for channel in results %}
|
{% for channel in results %}
|
||||||
<div class="channel-item {{ view_style }}">
|
<div class="channel-item {{ view_style }}">
|
||||||
<div class="channel-banner {{ view_style }}">
|
<div class="channel-banner {{ view_style }}">
|
||||||
<a href="{% url 'channel_id' channel.source.channel_id %}">
|
<a href="{% url 'channel_id' channel.channel_id %}">
|
||||||
<img src="/cache/channels/{{ channel.source.channel_id }}_banner.jpg" alt="{{ channel.source.channel_id }}-banner">
|
<img src="/cache/channels/{{ channel.channel_id }}_banner.jpg" alt="{{ channel.channel_id }}-banner">
|
||||||
</a>
|
</a>
|
||||||
</div>
|
</div>
|
||||||
<div class="info-box info-box-2 {{ view_style }}">
|
<div class="info-box info-box-2 {{ view_style }}">
|
||||||
<div class="info-box-item">
|
<div class="info-box-item">
|
||||||
<div class="round-img">
|
<div class="round-img">
|
||||||
<a href="{% url 'channel_id' channel.source.channel_id %}">
|
<a href="{% url 'channel_id' channel.channel_id %}">
|
||||||
<img src="/cache/channels/{{ channel.source.channel_id }}_thumb.jpg" alt="channel-thumb">
|
<img src="/cache/channels/{{ channel.channel_id }}_thumb.jpg" alt="channel-thumb">
|
||||||
</a>
|
</a>
|
||||||
</div>
|
</div>
|
||||||
<div>
|
<div>
|
||||||
<h3><a href="{% url 'channel_id' channel.source.channel_id %}">{{ channel.source.channel_name }}</a></h3>
|
<h3><a href="{% url 'channel_id' channel.channel_id %}">{{ channel.channel_name }}</a></h3>
|
||||||
{% if channel.source.channel_subs >= 1000000 %}
|
{% if channel.channel_subs >= 1000000 %}
|
||||||
<p>Subscribers: {{ channel.source.channel_subs|intword }}</p>
|
<p>Subscribers: {{ channel.channel_subs|intword }}</p>
|
||||||
{% else %}
|
{% else %}
|
||||||
<p>Subscribers: {{ channel.source.channel_subs|intcomma }}</p>
|
<p>Subscribers: {{ channel.channel_subs|intcomma }}</p>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="info-box-item">
|
<div class="info-box-item">
|
||||||
<div>
|
<div>
|
||||||
<p>Last refreshed: {{ channel.source.channel_last_refresh }}</p>
|
<p>Last refreshed: {{ channel.channel_last_refresh }}</p>
|
||||||
{% if channel.source.channel_subscribed %}
|
{% if channel.channel_subscribed %}
|
||||||
<button class="unsubscribe" type="button" id="{{ channel.source.channel_id }}" onclick="unsubscribe(this.id)" title="Unsubscribe from {{ channel.source.channel_name }}">Unsubscribe</button>
|
<button class="unsubscribe" type="button" data-type="channel" data-subscribe="" data-id="{{ channel.channel_id }}" onclick="subscribeStatus(this)" title="Unsubscribe from {{ channel.channel_name }}">Unsubscribe</button>
|
||||||
{% else %}
|
{% else %}
|
||||||
<button type="button" id="{{ channel.source.channel_id }}" onclick="subscribe(this.id)" title="Subscribe to {{ channel.source.channel_name }}">Subscribe</button>
|
<button type="button" data-type="channel" data-subscribe="true" data-id="{{ channel.channel_id }}" onclick="subscribeStatus(this)" title="Subscribe to {{ channel.channel_name }}">Subscribe</button>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
@ -2,11 +2,13 @@
|
||||||
{% block content %}
|
{% block content %}
|
||||||
{% load static %}
|
{% load static %}
|
||||||
{% load humanize %}
|
{% load humanize %}
|
||||||
|
{% load auth_extras %}
|
||||||
|
|
||||||
<div class="boxed-content">
|
<div class="boxed-content">
|
||||||
<div class="channel-banner">
|
<div class="channel-banner">
|
||||||
<a href="/channel/{{ channel_info.channel_id }}/"><img src="/cache/channels/{{ channel_info.channel_id }}_banner.jpg" alt="channel_banner"></a>
|
<a href="/channel/{{ channel_info.channel_id }}/"><img src="/cache/channels/{{ channel_info.channel_id }}_banner.jpg" alt="channel_banner"></a>
|
||||||
</div>
|
</div>
|
||||||
<div class="info-box-item channel-nav">
|
<div class="info-box-item child-page-nav">
|
||||||
<a href="{% url 'channel_id' channel_info.channel_id %}"><h3>Videos</h3></a>
|
<a href="{% url 'channel_id' channel_info.channel_id %}"><h3>Videos</h3></a>
|
||||||
{% if has_streams %}
|
{% if has_streams %}
|
||||||
<a href="{% url 'channel_id_live' channel_info.channel_id %}"><h3>Streams</h3></a>
|
<a href="{% url 'channel_id_live' channel_info.channel_id %}"><h3>Streams</h3></a>
|
||||||
|
@ -19,7 +21,9 @@
|
||||||
{% endif %}
|
{% endif %}
|
||||||
<a href="{% url 'channel_id_about' channel_info.channel_id %}"><h3>About</h3></a>
|
<a href="{% url 'channel_id_about' channel_info.channel_id %}"><h3>About</h3></a>
|
||||||
{% if has_pending %}
|
{% if has_pending %}
|
||||||
<a href="{% url 'downloads' %}?channel={{ channel_info.channel_id }}"><h3>Downloads</h3></a>
|
{% if request.user|has_group:"admin" or request.user.is_staff %}
|
||||||
|
<a href="{% url 'downloads' %}?channel={{ channel_info.channel_id }}"><h3>Downloads</h3></a>
|
||||||
|
{% endif %}
|
||||||
{% endif %}
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
<div id="notifications" data="channel reindex"></div>
|
<div id="notifications" data="channel reindex"></div>
|
||||||
|
@ -38,19 +42,22 @@
|
||||||
<p>Subscribers: {{ channel_info.channel_subs|intcomma }}</p>
|
<p>Subscribers: {{ channel_info.channel_subs|intcomma }}</p>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
{% if channel_info.channel_subscribed %}
|
{% if channel_info.channel_subscribed %}
|
||||||
<button class="unsubscribe" type="button" id="{{ channel_info.channel_id }}" onclick="unsubscribe(this.id)" title="Unsubscribe from {{ channel_info.channel_name }}">Unsubscribe</button>
|
{% if request.user|has_group:"admin" or request.user.is_staff %}
|
||||||
|
<button class="unsubscribe" type="button" data-type="channel" data-subscribe="" data-id="{{ channel_info.channel_id }}" onclick="subscribeStatus(this)" title="Unsubscribe from {{ channel_info.channel_name }}">Unsubscribe</button>
|
||||||
|
{% endif %}
|
||||||
{% else %}
|
{% else %}
|
||||||
<button type="button" id="{{ channel_info.channel_id }}" onclick="subscribe(this.id)" title="Subscribe to {{ channel_info.channel_name }}">Subscribe</button>
|
<button type="button" data-type="channel" data-subscribe="true" data-id="{{ channel_info.channel_id }}" onclick="subscribeStatus(this)" title="Subscribe to {{ channel_info.channel_name }}">Subscribe</button>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="info-box-item">
|
<div class="info-box-item">
|
||||||
<div>
|
{% if aggs %}
|
||||||
{% if max_hits %}
|
<p>{{ aggs.total_items.value }} videos <span class="space-carrot">|</span> {{ aggs.total_duration.value_str }} playback <span class="space-carrot">|</span> Total size {{ aggs.total_size.value|filesizeformat }}</p>
|
||||||
<p>Total Videos: {{ max_hits }}</p>
|
<div class="button-box">
|
||||||
<button title="Mark all videos from {{ channel_info.channel_name }} as watched" type="button" id="watched-button" data-id="{{ channel_info.channel_id }}" onclick="isWatchedButton(this)">Mark as watched</button>
|
<button title="Mark all videos from {{ channel_info.channel_name }} as watched" type="button" id="watched-button" data-id="{{ channel_info.channel_id }}" onclick="isWatchedButton(this)">Mark as watched</button>
|
||||||
{% endif %}
|
<button title="Mark all videos from {{ channel_info.channel_name }} as unwatched" type="button" id="unwatched-button" data-id="{{ channel_info.channel_id }}" onclick="isUnwatchedButton(this)">Mark as unwatched</button>
|
||||||
</div>
|
</div>
|
||||||
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
@ -70,13 +77,15 @@
|
||||||
<div class="sort">
|
<div class="sort">
|
||||||
<div id="hidden-form">
|
<div id="hidden-form">
|
||||||
<span>Sort by:</span>
|
<span>Sort by:</span>
|
||||||
<select name="sort" id="sort" onchange="sortChange(this.value)">
|
<select name="sort_by" id="sort" onchange="sortChange(this)">
|
||||||
<option value="published" {% if sort_by == "published" %}selected{% endif %}>date published</option>
|
<option value="published" {% if sort_by == "published" %}selected{% endif %}>date published</option>
|
||||||
<option value="downloaded" {% if sort_by == "downloaded" %}selected{% endif %}>date downloaded</option>
|
<option value="downloaded" {% if sort_by == "downloaded" %}selected{% endif %}>date downloaded</option>
|
||||||
<option value="views" {% if sort_by == "views" %}selected{% endif %}>views</option>
|
<option value="views" {% if sort_by == "views" %}selected{% endif %}>views</option>
|
||||||
<option value="likes" {% if sort_by == "likes" %}selected{% endif %}>likes</option>
|
<option value="likes" {% if sort_by == "likes" %}selected{% endif %}>likes</option>
|
||||||
|
<option value="duration" {% if sort_by == "duration" %}selected{% endif %}>duration</option>
|
||||||
|
<option value="filesize" {% if sort_by == "filesize" %}selected{% endif %}>file size</option>
|
||||||
</select>
|
</select>
|
||||||
<select name="sord-order" id="sort-order" onchange="sortChange(this.value)">
|
<select name="sort_order" id="sort-order" onchange="sortChange(this)">
|
||||||
<option value="asc" {% if sort_order == "asc" %}selected{% endif %}>asc</option>
|
<option value="asc" {% if sort_order == "asc" %}selected{% endif %}>asc</option>
|
||||||
<option value="desc" {% if sort_order == "desc" %}selected{% endif %}>desc</option>
|
<option value="desc" {% if sort_order == "desc" %}selected{% endif %}>desc</option>
|
||||||
</select>
|
</select>
|
||||||
|
@ -105,14 +114,14 @@
|
||||||
{% if results %}
|
{% if results %}
|
||||||
{% for video in results %}
|
{% for video in results %}
|
||||||
<div class="video-item {{ view_style }}">
|
<div class="video-item {{ view_style }}">
|
||||||
<a href="#player" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
|
<a href="#player" data-id="{{ video.youtube_id }}" onclick="createPlayer(this)">
|
||||||
<div class="video-thumb-wrap {{ view_style }}">
|
<div class="video-thumb-wrap {{ view_style }}">
|
||||||
<div class="video-thumb">
|
<div class="video-thumb">
|
||||||
<img src="{{ video.source.vid_thumb_url }}" alt="video-thumb">
|
<img src="{{ video.vid_thumb_url }}" alt="video-thumb">
|
||||||
{% if video.source.player.progress %}
|
{% if video.player.progress %}
|
||||||
<div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
|
<div class="video-progress-bar" id="progress-{{ video.youtube_id }}" style="width: {{video.player.progress}}%;"></div>
|
||||||
{% else %}
|
{% else %}
|
||||||
<div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: 0%;"></div>
|
<div class="video-progress-bar" id="progress-{{ video.youtube_id }}" style="width: 0%;"></div>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
<div class="video-play">
|
<div class="video-play">
|
||||||
|
@ -121,16 +130,16 @@
|
||||||
</div>
|
</div>
|
||||||
</a>
|
</a>
|
||||||
<div class="video-desc {{ view_style }}">
|
<div class="video-desc {{ view_style }}">
|
||||||
<div class="video-desc-player" id="video-info-{{ video.source.youtube_id }}">
|
<div class="video-desc-player" id="video-info-{{ video.youtube_id }}">
|
||||||
{% if video.source.player.watched %}
|
{% if video.player.watched %}
|
||||||
<img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" data-id="{{ video.source.youtube_id }}" data-status="watched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as unwatched">
|
<img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" data-id="{{ video.youtube_id }}" data-status="watched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as unwatched">
|
||||||
{% else %}
|
{% else %}
|
||||||
<img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" data-id="{{ video.source.youtube_id }}" data-status="unwatched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as watched">
|
<img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" data-id="{{ video.youtube_id }}" data-status="unwatched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as watched">
|
||||||
{% endif %}
|
{% endif %}
|
||||||
<span>{{ video.source.published }} | {{ video.source.player.duration_str }}</span>
|
<span>{{ video.published }} | {{ video.player.duration_str }}</span>
|
||||||
</div>
|
</div>
|
||||||
<div>
|
<div>
|
||||||
<a class="video-more" href="{% url 'video' video.source.youtube_id %}"><h2>{{ video.source.title }}</h2></a>
|
<a class="video-more" href="{% url 'video' video.youtube_id %}"><h2>{{ video.title }}</h2></a>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
@ -2,11 +2,12 @@
|
||||||
{% block content %}
|
{% block content %}
|
||||||
{% load static %}
|
{% load static %}
|
||||||
{% load humanize %}
|
{% load humanize %}
|
||||||
|
{% load auth_extras %}
|
||||||
<div class="boxed-content">
|
<div class="boxed-content">
|
||||||
<div class="channel-banner">
|
<div class="channel-banner">
|
||||||
<a href="{% url 'channel_id' channel_info.channel_id %}"><img src="{{ channel_info.channel_banner_url }}" alt="channel_banner"></a>
|
<a href="{% url 'channel_id' channel_info.channel_id %}"><img src="{{ channel_info.channel_banner_url }}" alt="channel_banner"></a>
|
||||||
</div>
|
</div>
|
||||||
<div class="info-box-item channel-nav">
|
<div class="info-box-item child-page-nav">
|
||||||
<a href="{% url 'channel_id' channel_info.channel_id %}"><h3>Videos</h3></a>
|
<a href="{% url 'channel_id' channel_info.channel_id %}"><h3>Videos</h3></a>
|
||||||
{% if has_streams %}
|
{% if has_streams %}
|
||||||
<a href="{% url 'channel_id_live' channel_info.channel_id %}"><h3>Streams</h3></a>
|
<a href="{% url 'channel_id_live' channel_info.channel_id %}"><h3>Streams</h3></a>
|
||||||
|
@ -19,7 +20,9 @@
|
||||||
{% endif %}
|
{% endif %}
|
||||||
<a href="{% url 'channel_id_about' channel_info.channel_id %}"><h3>About</h3></a>
|
<a href="{% url 'channel_id_about' channel_info.channel_id %}"><h3>About</h3></a>
|
||||||
{% if has_pending %}
|
{% if has_pending %}
|
||||||
<a href="{% url 'downloads' %}?channel={{ channel_info.channel_id }}"><h3>Downloads</h3></a>
|
{% if request.user|has_group:"admin" or request.user.is_staff %}
|
||||||
|
<a href="{% url 'downloads' %}?channel={{ channel_info.channel_id }}"><h3>Downloads</h3></a>
|
||||||
|
{% endif %}
|
||||||
{% endif %}
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
<div id="notifications" data="channel reindex"></div>
|
<div id="notifications" data="channel reindex"></div>
|
||||||
|
@ -56,19 +59,21 @@
|
||||||
{% elif channel_info.channel_views > 0 %}
|
{% elif channel_info.channel_views > 0 %}
|
||||||
<p>Channel views: {{ channel_info.channel_views|intcomma }}</p>
|
<p>Channel views: {{ channel_info.channel_views|intcomma }}</p>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
<div class="button-box">
|
{% if request.user|has_group:"admin" or request.user.is_staff %}
|
||||||
<button onclick="deleteConfirm()" id="delete-item">Delete Channel</button>
|
<div class="button-box">
|
||||||
<div class="delete-confirm" id="delete-button">
|
<button onclick="deleteConfirm()" id="delete-item">Delete Channel</button>
|
||||||
<span>Delete {{ channel_info.channel_name }} including all videos? </span><button class="danger-button" onclick="deleteChannel(this)" data-id="{{ channel_info.channel_id }}">Delete</button> <button onclick="cancelDelete()">Cancel</button>
|
<div class="delete-confirm" id="delete-button">
|
||||||
</div>
|
<span>Delete {{ channel_info.channel_name }} including all videos? </span><button class="danger-button" onclick="deleteChannel(this)" data-id="{{ channel_info.channel_id }}">Delete</button> <button onclick="cancelDelete()">Cancel</button>
|
||||||
</div>
|
</div>
|
||||||
{% if reindex %}
|
|
||||||
<p>Reindex scheduled</p>
|
|
||||||
{% else %}
|
|
||||||
<div id="reindex-button" class="button-box">
|
|
||||||
<button data-id="{{ channel_info.channel_id }}" data-type="channel" onclick="reindex(this)" title="Reindex Channel {{ channel_info.channel_name }}">Reindex</button>
|
|
||||||
<button data-id="{{ channel_info.channel_id }}" data-type="channel" data-extract-videos="true" onclick="reindex(this)" title="Reindex Videos of {{ channel_info.channel_name }}">Reindex Videos</button>
|
|
||||||
</div>
|
</div>
|
||||||
|
{% if reindex %}
|
||||||
|
<p>Reindex scheduled</p>
|
||||||
|
{% else %}
|
||||||
|
<div id="reindex-button" class="button-box">
|
||||||
|
<button data-id="{{ channel_info.channel_id }}" data-type="channel" onclick="reindex(this)" title="Reindex Channel {{ channel_info.channel_name }}">Reindex</button>
|
||||||
|
<button data-id="{{ channel_info.channel_id }}" data-type="channel" data-extract-videos="true" onclick="reindex(this)" title="Reindex Videos of {{ channel_info.channel_name }}">Reindex Videos</button>
|
||||||
|
</div>
|
||||||
|
{% endif %}
|
||||||
{% endif %}
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
@ -81,55 +86,64 @@
|
||||||
<button onclick="textExpand()" id="text-expand-button">Show more</button>
|
<button onclick="textExpand()" id="text-expand-button">Show more</button>
|
||||||
</div>
|
</div>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
<div class="description-box">
|
{% if channel_info.channel_tags %}
|
||||||
<h2>Customize {{ channel_info.channel_name }}</h2>
|
<div class="description-box">
|
||||||
</div>
|
<div class="video-tag-box">
|
||||||
<div id="overwrite-form" class="info-box">
|
{% for tag in channel_info.channel_tags %}
|
||||||
<div class="info-box-item">
|
<span class="video-tag">{{ tag }}</span>
|
||||||
<form class="overwrite-form" action="/channel/{{ channel_info.channel_id }}/about/" method="POST">
|
{% endfor %}
|
||||||
{% csrf_token %}
|
</div>
|
||||||
<div class="overwrite-form-item">
|
|
||||||
<p>Download format: <span class="settings-current">
|
|
||||||
{% if channel_info.channel_overwrites.download_format %}
|
|
||||||
{{ channel_info.channel_overwrites.download_format }}
|
|
||||||
{% else %}
|
|
||||||
False
|
|
||||||
{% endif %}</span></p>
|
|
||||||
{{ channel_overwrite_form.download_format }}<br>
|
|
||||||
</div>
|
|
||||||
<div class="overwrite-form-item">
|
|
||||||
<p>Auto delete watched videos after x days: <span class="settings-current">
|
|
||||||
{% if channel_info.channel_overwrites.autodelete_days %}
|
|
||||||
{{ channel_info.channel_overwrites.autodelete_days }}
|
|
||||||
{% else %}
|
|
||||||
False
|
|
||||||
{% endif %}</span></p>
|
|
||||||
{{ channel_overwrite_form.autodelete_days }}<br>
|
|
||||||
</div>
|
|
||||||
<div class="overwrite-form-item">
|
|
||||||
<p>Index playlists: <span class="settings-current">
|
|
||||||
{% if channel_info.channel_overwrites.index_playlists %}
|
|
||||||
{{ channel_info.channel_overwrites.index_playlists }}
|
|
||||||
{% else %}
|
|
||||||
False
|
|
||||||
{% endif %}</span></p>
|
|
||||||
{{ channel_overwrite_form.index_playlists }}<br>
|
|
||||||
</div>
|
|
||||||
<div class="overwrite-form-item">
|
|
||||||
<p>Enable <a href="https://sponsor.ajay.app/" target="_blank">SponsorBlock</a>: <span class="settings-current">
|
|
||||||
{% if channel_info.channel_overwrites.integrate_sponsorblock %}
|
|
||||||
{{ channel_info.channel_overwrites.integrate_sponsorblock }}
|
|
||||||
{% elif channel_info.channel_overwrites.integrate_sponsorblock == False %}
|
|
||||||
Disabled
|
|
||||||
{% else %}
|
|
||||||
False
|
|
||||||
{% endif %}</span></p>
|
|
||||||
{{ channel_overwrite_form.integrate_sponsorblock }}<br>
|
|
||||||
</div>
|
|
||||||
<button type="submit">Save Channel Overwrites</button>
|
|
||||||
</form>
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
{% endif %}
|
||||||
|
{% if request.user|has_group:"admin" or request.user.is_staff %}
|
||||||
|
<div id="overwrite-form" class="info-box">
|
||||||
|
<div class="info-box-item">
|
||||||
|
<h2>Customize {{ channel_info.channel_name }}</h2>
|
||||||
|
<form class="overwrite-form" action="/channel/{{ channel_info.channel_id }}/about/" method="POST">
|
||||||
|
{% csrf_token %}
|
||||||
|
<div class="overwrite-form-item">
|
||||||
|
<p>Download format: <span class="settings-current">
|
||||||
|
{% if channel_info.channel_overwrites.download_format %}
|
||||||
|
{{ channel_info.channel_overwrites.download_format }}
|
||||||
|
{% else %}
|
||||||
|
False
|
||||||
|
{% endif %}</span></p>
|
||||||
|
{{ channel_overwrite_form.download_format }}<br>
|
||||||
|
</div>
|
||||||
|
<div class="overwrite-form-item">
|
||||||
|
<p>Auto delete watched videos after x days: <span class="settings-current">
|
||||||
|
{% if channel_info.channel_overwrites.autodelete_days %}
|
||||||
|
{{ channel_info.channel_overwrites.autodelete_days }}
|
||||||
|
{% else %}
|
||||||
|
False
|
||||||
|
{% endif %}</span></p>
|
||||||
|
{{ channel_overwrite_form.autodelete_days }}<br>
|
||||||
|
</div>
|
||||||
|
<div class="overwrite-form-item">
|
||||||
|
<p>Index playlists: <span class="settings-current">
|
||||||
|
{% if channel_info.channel_overwrites.index_playlists %}
|
||||||
|
{{ channel_info.channel_overwrites.index_playlists }}
|
||||||
|
{% else %}
|
||||||
|
False
|
||||||
|
{% endif %}</span></p>
|
||||||
|
{{ channel_overwrite_form.index_playlists }}<br>
|
||||||
|
</div>
|
||||||
|
<div class="overwrite-form-item">
|
||||||
|
<p>Enable <a href="https://sponsor.ajay.app/" target="_blank">SponsorBlock</a>: <span class="settings-current">
|
||||||
|
{% if channel_info.channel_overwrites.integrate_sponsorblock %}
|
||||||
|
{{ channel_info.channel_overwrites.integrate_sponsorblock }}
|
||||||
|
{% elif channel_info.channel_overwrites.integrate_sponsorblock == False %}
|
||||||
|
Disabled
|
||||||
|
{% else %}
|
||||||
|
False
|
||||||
|
{% endif %}</span></p>
|
||||||
|
{{ channel_overwrite_form.integrate_sponsorblock }}<br>
|
||||||
|
</div>
|
||||||
|
<button type="submit">Save Channel Overwrites</button>
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
<script type="text/javascript" src="{% static 'progress.js' %}"></script>
|
<script type="text/javascript" src="{% static 'progress.js' %}"></script>
|
||||||
{% endblock content %}
|
{% endblock content %}
|
|
@ -2,11 +2,12 @@
|
||||||
{% block content %}
|
{% block content %}
|
||||||
{% load static %}
|
{% load static %}
|
||||||
{% load humanize %}
|
{% load humanize %}
|
||||||
|
{% load auth_extras %}
|
||||||
<div class="boxed-content">
|
<div class="boxed-content">
|
||||||
<div class="channel-banner">
|
<div class="channel-banner">
|
||||||
<a href="{% url 'channel_id' channel_info.channel_id %}"><img src="{{ channel_info.channel_banner_url }}" alt="channel_banner"></a>
|
<a href="{% url 'channel_id' channel_info.channel_id %}"><img src="{{ channel_info.channel_banner_url }}" alt="channel_banner"></a>
|
||||||
</div>
|
</div>
|
||||||
<div class="info-box-item channel-nav">
|
<div class="info-box-item child-page-nav">
|
||||||
<a href="{% url 'channel_id' channel_info.channel_id %}"><h3>Videos</h3></a>
|
<a href="{% url 'channel_id' channel_info.channel_id %}"><h3>Videos</h3></a>
|
||||||
{% if has_streams %}
|
{% if has_streams %}
|
||||||
<a href="{% url 'channel_id_live' channel_info.channel_id %}"><h3>Streams</h3></a>
|
<a href="{% url 'channel_id_live' channel_info.channel_id %}"><h3>Streams</h3></a>
|
||||||
|
@ -19,7 +20,9 @@
|
||||||
{% endif %}
|
{% endif %}
|
||||||
<a href="{% url 'channel_id_about' channel_info.channel_id %}"><h3>About</h3></a>
|
<a href="{% url 'channel_id_about' channel_info.channel_id %}"><h3>About</h3></a>
|
||||||
{% if has_pending %}
|
{% if has_pending %}
|
||||||
<a href="{% url 'downloads' %}?channel={{ channel_info.channel_id }}"><h3>Downloads</h3></a>
|
{% if request.user|has_group:"admin" or request.user.is_staff %}
|
||||||
|
<a href="{% url 'downloads' %}?channel={{ channel_info.channel_id }}"><h3>Downloads</h3></a>
|
||||||
|
{% endif %}
|
||||||
{% endif %}
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
<div id="notifications" data="channel reindex"></div>
|
<div id="notifications" data="channel reindex"></div>
|
||||||
|
@ -45,18 +48,19 @@
|
||||||
{% for playlist in results %}
|
{% for playlist in results %}
|
||||||
<div class="playlist-item {{ view_style }}">
|
<div class="playlist-item {{ view_style }}">
|
||||||
<div class="playlist-thumbnail">
|
<div class="playlist-thumbnail">
|
||||||
<a href="{% url 'playlist_id' playlist.source.playlist_id %}">
|
<a href="{% url 'playlist_id' playlist.playlist_id %}">
|
||||||
<img src="/cache/playlists/{{ playlist.source.playlist_id }}.jpg" alt="{{ playlist.source.playlist_id }}-thumbnail">
|
<img src="/cache/playlists/{{ playlist.playlist_id }}.jpg" alt="{{ playlist.playlist_id }}-thumbnail">
|
||||||
</a>
|
</a>
|
||||||
</div>
|
</div>
|
||||||
<div class="playlist-desc {{ view_style }}">
|
<div class="playlist-desc {{ view_style }}">
|
||||||
<a href="{% url 'channel_id' playlist.source.playlist_channel_id %}"><h3>{{ playlist.source.playlist_channel }}</h3></a>
|
<a href="{% url 'playlist_id' playlist.playlist_id %}"><h2>{{ playlist.playlist_name }}</h2></a>
|
||||||
<a href="{% url 'playlist_id' playlist.source.playlist_id %}"><h2>{{ playlist.source.playlist_name }}</h2></a>
|
<p>Last refreshed: {{ playlist.playlist_last_refresh }}</p>
|
||||||
<p>Last refreshed: {{ playlist.source.playlist_last_refresh }}</p>
|
{% if request.user|has_group:"admin" or request.user.is_staff %}
|
||||||
{% if playlist.source.playlist_subscribed %}
|
{% if playlist.playlist_subscribed %}
|
||||||
<button class="unsubscribe" type="button" id="{{ playlist.source.playlist_id }}" onclick="unsubscribe(this.id)" title="Unsubscribe from {{ playlist.source.playlist_name }}">Unsubscribe</button>
|
<button class="unsubscribe" type="button" data-type="playlist" data-subscribe="" data-id="{{ playlist.playlist_id }}" onclick="subscribeStatus(this)" title="Unsubscribe from {{ playlist.playlist_name }}">Unsubscribe</button>
|
||||||
{% else %}
|
{% else %}
|
||||||
<button type="button" id="{{ playlist.source.playlist_id }}" onclick="subscribe(this.id)" title="Subscribe to {{ playlist.source.playlist_name }}">Subscribe</button>
|
<button type="button" data-type="playlist" data-subscribe="true" data-id="{{ playlist.playlist_id }}" onclick="subscribeStatus(this)" title="Subscribe to {{ playlist.playlist_name }}">Subscribe</button>
|
||||||
|
{% endif %}
|
||||||
{% endif %}
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
@ -20,11 +20,12 @@
|
||||||
<img id="animate-icon" onclick="showForm()" src="{% static 'img/icon-add.svg' %}" alt="add-icon">
|
<img id="animate-icon" onclick="showForm()" src="{% static 'img/icon-add.svg' %}" alt="add-icon">
|
||||||
<p>Add to download queue</p>
|
<p>Add to download queue</p>
|
||||||
<div class="show-form">
|
<div class="show-form">
|
||||||
<form id='hidden-form' action="/downloads/" method="post">
|
<div id='hidden-form' novalidate>
|
||||||
{% csrf_token %}
|
{% csrf_token %}
|
||||||
{{ add_form }}
|
{{ add_form }}
|
||||||
<button type="submit">Add to download queue</button>
|
<button onclick="addToQueue()">Add to queue</button>
|
||||||
</form>
|
<button onclick="addToQueue(true)">Download now</button>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
@ -69,37 +70,46 @@
|
||||||
<div class="video-list {{ view_style }} {% if view_style == "grid" %}grid-{{ grid_items }}{% endif %}">
|
<div class="video-list {{ view_style }} {% if view_style == "grid" %}grid-{{ grid_items }}{% endif %}">
|
||||||
{% if results %}
|
{% if results %}
|
||||||
{% for video in results %}
|
{% for video in results %}
|
||||||
<div class="video-item {{ view_style }}" id="dl-{{ video.source.youtube_id }}">
|
<div class="video-item {{ view_style }}" id="dl-{{ video.youtube_id }}">
|
||||||
<div class="video-thumb-wrap {{ view_style }}">
|
<div class="video-thumb-wrap {{ view_style }}">
|
||||||
<div class="video-thumb">
|
<div class="video-thumb">
|
||||||
<img src="{{ video.source.vid_thumb_url }}" alt="video_thumb">
|
<img src="{{ video.vid_thumb_url }}" alt="video_thumb">
|
||||||
<div class="video-tags">
|
<div class="video-tags">
|
||||||
{% if show_ignored_only %}
|
{% if show_ignored_only %}
|
||||||
<span>ignored</span>
|
<span>ignored</span>
|
||||||
{% else %}
|
{% else %}
|
||||||
<span>queued</span>
|
<span>queued</span>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
<span>{{ video.source.vid_type }}</span>
|
<span>{{ video.vid_type }}</span>
|
||||||
|
{% if video.auto_start %}
|
||||||
|
<span>auto</span>
|
||||||
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="video-desc {{ view_style }}">
|
<div class="video-desc {{ view_style }}">
|
||||||
<div>
|
<div>
|
||||||
{% if video.source.channel_indexed %}
|
{% if video.channel_indexed %}
|
||||||
<a href="{% url 'channel_id' video.source.channel_id %}">{{ video.source.channel_name }}</a>
|
<a href="{% url 'channel_id' video.channel_id %}">{{ video.channel_name }}</a>
|
||||||
{% else %}
|
{% else %}
|
||||||
<span>{{ video.source.channel_name }}</span>
|
<span>{{ video.channel_name }}</span>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
<a href="https://www.youtube.com/watch?v={{ video.source.youtube_id }}" target="_blank"><h3>{{ video.source.title }}</h3></a>
|
<a href="https://www.youtube.com/watch?v={{ video.youtube_id }}" target="_blank"><h3>{{ video.title }}</h3></a>
|
||||||
</div>
|
</div>
|
||||||
<p>Published: {{ video.source.published }} | Duration: {{ video.source.duration }} | {{ video.source.youtube_id }}</p>
|
<p>Published: {{ video.published }} | Duration: {{ video.duration }} | {{ video.youtube_id }}</p>
|
||||||
|
{% if video.message %}
|
||||||
|
<p class="danger-zone">{{ video.message }}</p>
|
||||||
|
{% endif %}
|
||||||
<div>
|
<div>
|
||||||
{% if show_ignored_only %}
|
{% if show_ignored_only %}
|
||||||
<button data-id="{{ video.source.youtube_id }}" onclick="forgetIgnore(this)">Forget</button>
|
<button data-id="{{ video.youtube_id }}" onclick="forgetIgnore(this)">Forget</button>
|
||||||
<button data-id="{{ video.source.youtube_id }}" onclick="addSingle(this)">Add to queue</button>
|
<button data-id="{{ video.youtube_id }}" onclick="addSingle(this)">Add to queue</button>
|
||||||
{% else %}
|
{% else %}
|
||||||
<button data-id="{{ video.source.youtube_id }}" onclick="toIgnore(this)">Ignore</button>
|
<button data-id="{{ video.youtube_id }}" onclick="toIgnore(this)">Ignore</button>
|
||||||
<button id="{{ video.source.youtube_id }}" data-id="{{ video.source.youtube_id }}" onclick="downloadNow(this)">Download now</button>
|
<button id="{{ video.youtube_id }}" data-id="{{ video.youtube_id }}" onclick="downloadNow(this)">Download now</button>
|
||||||
|
{% endif %}
|
||||||
|
{% if video.message %}
|
||||||
|
<button class="danger-button" data-id="{{ video.youtube_id }}" onclick="forgetIgnore(this)">Delete</button>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
@ -9,14 +9,14 @@
|
||||||
<div class="video-list {{ view_style }} {% if view_style == "grid" %}grid-{{ grid_items }}{% endif %}">
|
<div class="video-list {{ view_style }} {% if view_style == "grid" %}grid-{{ grid_items }}{% endif %}">
|
||||||
{% for video in continue_vids %}
|
{% for video in continue_vids %}
|
||||||
<div class="video-item {{ view_style }}">
|
<div class="video-item {{ view_style }}">
|
||||||
<a href="#player" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
|
<a href="#player" data-id="{{ video.youtube_id }}" onclick="createPlayer(this)">
|
||||||
<div class="video-thumb-wrap {{ view_style }}">
|
<div class="video-thumb-wrap {{ view_style }}">
|
||||||
<div class="video-thumb">
|
<div class="video-thumb">
|
||||||
<img src="{{ video.source.vid_thumb_url }}" alt="video-thumb">
|
<img src="{{ video.vid_thumb_url }}" alt="video-thumb">
|
||||||
{% if video.source.player.progress %}
|
{% if video.player.progress %}
|
||||||
<div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
|
<div class="video-progress-bar" id="progress-{{ video.youtube_id }}" style="width: {{video.player.progress}}%;"></div>
|
||||||
{% else %}
|
{% else %}
|
||||||
<div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: 0%;"></div>
|
<div class="video-progress-bar" id="progress-{{ video.youtube_id }}" style="width: 0%;"></div>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
<div class="video-play">
|
<div class="video-play">
|
||||||
|
@ -25,17 +25,17 @@
|
||||||
</div>
|
</div>
|
||||||
</a>
|
</a>
|
||||||
<div class="video-desc {{ view_style }}">
|
<div class="video-desc {{ view_style }}">
|
||||||
<div class="video-desc-player" id="video-info-{{ video.source.youtube_id }}">
|
<div class="video-desc-player" id="video-info-{{ video.youtube_id }}">
|
||||||
{% if video.source.player.watched %}
|
{% if video.player.watched %}
|
||||||
<img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" data-id="{{ video.source.youtube_id }}" data-status="watched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as unwatched">
|
<img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" data-id="{{ video.youtube_id }}" data-status="watched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as unwatched">
|
||||||
{% else %}
|
{% else %}
|
||||||
<img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" data-id="{{ video.source.youtube_id }}" data-status="unwatched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as watched">
|
<img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" data-id="{{ video.youtube_id }}" data-status="unwatched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as watched">
|
||||||
{% endif %}
|
{% endif %}
|
||||||
<span>{{ video.source.published }} | {{ video.source.player.duration_str }}</span>
|
<span>{{ video.published }} | {{ video.player.duration_str }}</span>
|
||||||
</div>
|
</div>
|
||||||
<div>
|
<div>
|
||||||
<a href="{% url 'channel_id' video.source.channel.channel_id %}"><h3>{{ video.source.channel.channel_name }}</h3></a>
|
<a href="{% url 'channel_id' video.channel.channel_id %}"><h3>{{ video.channel.channel_name }}</h3></a>
|
||||||
<a class="video-more" href="{% url 'video' video.source.youtube_id %}"><h2>{{ video.source.title }}</h2></a>
|
<a class="video-more" href="{% url 'video' video.youtube_id %}"><h2>{{ video.title }}</h2></a>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
@ -60,13 +60,15 @@
|
||||||
<div class="sort">
|
<div class="sort">
|
||||||
<div id="hidden-form">
|
<div id="hidden-form">
|
||||||
<span>Sort by:</span>
|
<span>Sort by:</span>
|
||||||
<select name="sort" id="sort" onchange="sortChange(this.value)">
|
<select name="sort_by" id="sort" onchange="sortChange(this)">
|
||||||
<option value="published" {% if sort_by == "published" %}selected{% endif %}>date published</option>
|
<option value="published" {% if sort_by == "published" %}selected{% endif %}>date published</option>
|
||||||
<option value="downloaded" {% if sort_by == "downloaded" %}selected{% endif %}>date downloaded</option>
|
<option value="downloaded" {% if sort_by == "downloaded" %}selected{% endif %}>date downloaded</option>
|
||||||
<option value="views" {% if sort_by == "views" %}selected{% endif %}>views</option>
|
<option value="views" {% if sort_by == "views" %}selected{% endif %}>views</option>
|
||||||
<option value="likes" {% if sort_by == "likes" %}selected{% endif %}>likes</option>
|
<option value="likes" {% if sort_by == "likes" %}selected{% endif %}>likes</option>
|
||||||
|
<option value="duration" {% if sort_by == "duration" %}selected{% endif %}>duration</option>
|
||||||
|
<option value="filesize" {% if sort_by == "filesize" %}selected{% endif %}>file size</option>
|
||||||
</select>
|
</select>
|
||||||
<select name="sord-order" id="sort-order" onchange="sortChange(this.value)">
|
<select name="sort_order" id="sort-order" onchange="sortChange(this)">
|
||||||
<option value="asc" {% if sort_order == "asc" %}selected{% endif %}>asc</option>
|
<option value="asc" {% if sort_order == "asc" %}selected{% endif %}>asc</option>
|
||||||
<option value="desc" {% if sort_order == "desc" %}selected{% endif %}>desc</option>
|
<option value="desc" {% if sort_order == "desc" %}selected{% endif %}>desc</option>
|
||||||
</select>
|
</select>
|
||||||
|
@ -95,14 +97,14 @@
|
||||||
{% if results %}
|
{% if results %}
|
||||||
{% for video in results %}
|
{% for video in results %}
|
||||||
<div class="video-item {{ view_style }}">
|
<div class="video-item {{ view_style }}">
|
||||||
<a href="#player" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
|
<a href="#player" data-id="{{ video.youtube_id }}" onclick="createPlayer(this)">
|
||||||
<div class="video-thumb-wrap {{ view_style }}">
|
<div class="video-thumb-wrap {{ view_style }}">
|
||||||
<div class="video-thumb">
|
<div class="video-thumb">
|
||||||
<img src="{{ video.source.vid_thumb_url }}" alt="video-thumb">
|
<img src="{{ video.vid_thumb_url }}" alt="video-thumb">
|
||||||
{% if video.source.player.progress %}
|
{% if video.player.progress %}
|
||||||
<div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
|
<div class="video-progress-bar" id="progress-{{ video.youtube_id }}" style="width: {{video.player.progress}}%;"></div>
|
||||||
{% else %}
|
{% else %}
|
||||||
<div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: 0%;"></div>
|
<div class="video-progress-bar" id="progress-{{ video.youtube_id }}" style="width: 0%;"></div>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
<div class="video-play">
|
<div class="video-play">
|
||||||
|
@ -111,17 +113,17 @@
|
||||||
</div>
|
</div>
|
||||||
</a>
|
</a>
|
||||||
<div class="video-desc {{ view_style }}">
|
<div class="video-desc {{ view_style }}">
|
||||||
<div class="video-desc-player" id="video-info-{{ video.source.youtube_id }}">
|
<div class="video-desc-player" id="video-info-{{ video.youtube_id }}">
|
||||||
{% if video.source.player.watched %}
|
{% if video.player.watched %}
|
||||||
<img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" data-id="{{ video.source.youtube_id }}" data-status="watched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as unwatched">
|
<img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" data-id="{{ video.youtube_id }}" data-status="watched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as unwatched">
|
||||||
{% else %}
|
{% else %}
|
||||||
<img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" data-id="{{ video.source.youtube_id }}" data-status="unwatched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as watched">
|
<img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" data-id="{{ video.youtube_id }}" data-status="unwatched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as watched">
|
||||||
{% endif %}
|
{% endif %}
|
||||||
<span>{{ video.source.published }} | {{ video.source.player.duration_str }}</span>
|
<span>{{ video.published }} | {{ video.player.duration_str }}</span>
|
||||||
</div>
|
</div>
|
||||||
<div>
|
<div>
|
||||||
<a href="{% url 'channel_id' video.source.channel.channel_id %}"><h3>{{ video.source.channel.channel_name }}</h3></a>
|
<a href="{% url 'channel_id' video.channel.channel_id %}"><h3>{{ video.channel.channel_name }}</h3></a>
|
||||||
<a class="video-more" href="{% url 'video' video.source.youtube_id %}"><h2>{{ video.source.title }}</h2></a>
|
<a class="video-more" href="{% url 'video' video.youtube_id %}"><h2>{{ video.title }}</h2></a>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
@ -18,20 +18,11 @@
|
||||||
<meta name="msapplication-TileColor" content="#01202e">
|
<meta name="msapplication-TileColor" content="#01202e">
|
||||||
<meta name="msapplication-config" content="{% static 'favicon/browserconfig.xml' %}">
|
<meta name="msapplication-config" content="{% static 'favicon/browserconfig.xml' %}">
|
||||||
<meta name="theme-color" content="#01202e">
|
<meta name="theme-color" content="#01202e">
|
||||||
{% if colors == "dark" %}
|
<link rel="stylesheet" href="{% static 'css/' %}{{ stylesheet }}">
|
||||||
<link rel="stylesheet" href="{% static 'css/dark.css' %}">
|
|
||||||
{% else %}
|
|
||||||
<link rel="stylesheet" href="{% static 'css/light.css' %}">
|
|
||||||
{% endif %}
|
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
<div class="boxed-content login-page">
|
<div class="boxed-content login-page">
|
||||||
{% if colors == 'dark' %}
|
<img alt="tube-archivist-logo">
|
||||||
<img src="{% static 'img/logo-tube-archivist-dark.png' %}" alt="tube-archivist-logo">
|
|
||||||
{% endif %}
|
|
||||||
{% if colors == 'light' %}
|
|
||||||
<img src="{% static 'img/logo-tube-archivist-light.png' %}" alt="tube-archivist-banner">
|
|
||||||
{% endif %}
|
|
||||||
<h1>Tube Archivist</h1>
|
<h1>Tube Archivist</h1>
|
||||||
<h2>Your Self Hosted YouTube Media Server</h2>
|
<h2>Your Self Hosted YouTube Media Server</h2>
|
||||||
{% if form_error %}
|
{% if form_error %}
|
||||||
|
|
|
@ -1,21 +1,32 @@
|
||||||
{% extends "home/base.html" %}
|
{% extends "home/base.html" %}
|
||||||
{% load static %}
|
{% load static %}
|
||||||
{% block content %}
|
{% block content %}
|
||||||
|
{% load auth_extras %}
|
||||||
|
|
||||||
<div class="boxed-content">
|
<div class="boxed-content">
|
||||||
<div class="title-split">
|
<div class="title-split">
|
||||||
<div class="title-bar">
|
<div class="title-bar">
|
||||||
<h1>Playlists</h1>
|
<h1>Playlists</h1>
|
||||||
</div>
|
</div>
|
||||||
|
{% if request.user|has_group:"admin" or request.user.is_staff %}
|
||||||
|
|
||||||
<div class="title-split-form">
|
<div class="title-split-form">
|
||||||
<img id="animate-icon" onclick="showForm()" src="{% static 'img/icon-add.svg' %}" alt="add-icon" title="Subscribe to Playlists">
|
<img id="animate-icon" onclick="showForm();showForm('hidden-form2')" src="{% static 'img/icon-add.svg' %}" alt="add-icon" title="Subscribe to Playlists">
|
||||||
<div class="show-form">
|
<div class="show-form">
|
||||||
<form id="hidden-form" action="/playlist/" method="post">
|
<form id="hidden-form" action="/playlist/" method="post">
|
||||||
{% csrf_token %}
|
{% csrf_token %}
|
||||||
{{ subscribe_form }}
|
{{ subscribe_form }}
|
||||||
<button type="submit">Subscribe</button>
|
<button type="submit">Subscribe</button>
|
||||||
</form>
|
</form>
|
||||||
|
<form id="hidden-form2" action="/playlist/" method="post">
|
||||||
|
{% csrf_token %}
|
||||||
|
{{ create_form }}
|
||||||
|
<button type="submit">Create</button>
|
||||||
|
</form>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
<div id="notifications" data="subscription"></div>
|
<div id="notifications" data="subscription"></div>
|
||||||
<div class="view-controls">
|
<div class="view-controls">
|
||||||
|
@ -40,19 +51,23 @@
|
||||||
{% for playlist in results %}
|
{% for playlist in results %}
|
||||||
<div class="playlist-item {{ view_style }}">
|
<div class="playlist-item {{ view_style }}">
|
||||||
<div class="playlist-thumbnail">
|
<div class="playlist-thumbnail">
|
||||||
<a href="{% url 'playlist_id' playlist.source.playlist_id %}">
|
<a href="{% url 'playlist_id' playlist.playlist_id %}">
|
||||||
<img src="/cache/playlists/{{ playlist.source.playlist_id }}.jpg" alt="{{ playlist.source.playlist_id }}-thumbnail">
|
<img src="/cache/playlists/{{ playlist.playlist_id }}.jpg" alt="{{ playlist.playlist_id }}-thumbnail">
|
||||||
</a>
|
</a>
|
||||||
</div>
|
</div>
|
||||||
<div class="playlist-desc {{ view_style }}">
|
<div class="playlist-desc {{ view_style }}">
|
||||||
<a href="{% url 'channel_id' playlist.source.playlist_channel_id %}"><h3>{{ playlist.source.playlist_channel }}</h3></a>
|
{% if playlist.playlist_type != "custom" %}
|
||||||
<a href="{% url 'playlist_id' playlist.source.playlist_id %}"><h2>{{ playlist.source.playlist_name }}</h2></a>
|
<a href="{% url 'channel_id' playlist.playlist_channel_id %}"><h3>{{ playlist.playlist_channel }}</h3></a>
|
||||||
<p>Last refreshed: {{ playlist.source.playlist_last_refresh }}</p>
|
|
||||||
{% if playlist.source.playlist_subscribed %}
|
|
||||||
<button class="unsubscribe" type="button" id="{{ playlist.source.playlist_id }}" onclick="unsubscribe(this.id)" title="Unsubscribe from {{ playlist.source.playlist_name }}">Unsubscribe</button>
|
|
||||||
{% else %}
|
|
||||||
<button type="button" id="{{ playlist.source.playlist_id }}" onclick="subscribe(this.id)" title="Subscribe to {{ playlist.source.playlist_name }}">Subscribe</button>
|
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
<a href="{% url 'playlist_id' playlist.playlist_id %}"><h2>{{ playlist.playlist_name }}</h2></a>
|
||||||
|
<p>Last refreshed: {{ playlist.playlist_last_refresh }}</p>
|
||||||
|
{% if playlist.playlist_type != "custom" %}
|
||||||
|
{% if playlist.playlist_subscribed %}
|
||||||
|
<button class="unsubscribe" type="button" data-type="playlist" data-subscribe="" data-id="{{ playlist.playlist_id }}" onclick="subscribeStatus(this)" title="Unsubscribe from {{ playlist.playlist_name }}">Unsubscribe</button>
|
||||||
|
{% else %}
|
||||||
|
<button type="button" data-type="playlist" data-subscribe="true" data-id="{{ playlist.playlist_id }}" onclick="subscribeStatus(this)" title="Subscribe to {{ playlist.playlist_name }}">Subscribe</button>
|
||||||
|
{% endif %}
|
||||||
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
|
|
|
@ -2,46 +2,55 @@
|
||||||
{% load static %}
|
{% load static %}
|
||||||
{% load humanize %}
|
{% load humanize %}
|
||||||
{% block content %}
|
{% block content %}
|
||||||
|
{% load auth_extras %}
|
||||||
|
|
||||||
<div class="boxed-content">
|
<div class="boxed-content">
|
||||||
<div class="title-bar">
|
<div class="title-bar">
|
||||||
<h1>{{ playlist_info.playlist_name }}</h1>
|
<h1>{{ playlist_info.playlist_name }}</h1>
|
||||||
</div>
|
</div>
|
||||||
<div class="info-box info-box-3">
|
<div class="info-box info-box-3">
|
||||||
<div class="info-box-item">
|
{% if playlist_info.playlist_type != "custom" %}
|
||||||
<div class="round-img">
|
<div class="info-box-item">
|
||||||
<a href="{% url 'channel_id' channel_info.channel_id %}">
|
<div class="round-img">
|
||||||
<img src="/cache/channels/{{ channel_info.channel_id }}_thumb.jpg" alt="channel-thumb">
|
<a href="{% url 'channel_id' channel_info.channel_id %}">
|
||||||
</a>
|
<img src="/cache/channels/{{ channel_info.channel_id }}_thumb.jpg" alt="channel-thumb">
|
||||||
</div>
|
</a>
|
||||||
<div>
|
</div>
|
||||||
<h3><a href="{% url 'channel_id' channel_info.channel_id %}">{{ channel_info.channel_name }}</a></h3>
|
<div>
|
||||||
{% if channel_info.channel_subs >= 1000000 %}
|
<h3><a href="{% url 'channel_id' channel_info.channel_id %}">{{ channel_info.channel_name }}</a></h3>
|
||||||
<span>Subscribers: {{ channel_info.channel_subs|intword }}</span>
|
{% if channel_info.channel_subs >= 1000000 %}
|
||||||
{% else %}
|
<span>Subscribers: {{ channel_info.channel_subs|intword }}</span>
|
||||||
<span>Subscribers: {{ channel_info.channel_subs|intcomma }}</span>
|
{% else %}
|
||||||
{% endif %}
|
<span>Subscribers: {{ channel_info.channel_subs|intcomma }}</span>
|
||||||
</div>
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
|
</div>
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
<div class="info-box-item">
|
<div class="info-box-item">
|
||||||
<div>
|
<div>
|
||||||
<p>Last refreshed: {{ playlist_info.playlist_last_refresh }}</p>
|
<p>Last refreshed: {{ playlist_info.playlist_last_refresh }}</p>
|
||||||
<p>Playlist:
|
{% if playlist_info.playlist_type != "custom" %}
|
||||||
{% if playlist_info.playlist_subscribed %}
|
<p>Playlist:
|
||||||
<button class="unsubscribe" type="button" id="{{ playlist_info.playlist_id }}" onclick="unsubscribe(this.id)" title="Unsubscribe from {{ playlist_info.playlist_name }}">Unsubscribe</button>
|
{% if playlist_info.playlist_subscribed %}
|
||||||
{% else %}
|
{% if request.user|has_group:"admin" or request.user.is_staff %}
|
||||||
<button type="button" id="{{ playlist_info.playlist_id }}" onclick="subscribe(this.id)" title="Subscribe to {{ playlist_info.playlist_name }}">Subscribe</button>
|
<button class="unsubscribe" type="button" data-type="playlist" data-subscribe="" data-id="{{ playlist_info.playlist_id }}" onclick="subscribeStatus(this)" title="Unsubscribe from {{ playlist_info.playlist_name }}">Unsubscribe</button>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
</p>
|
{% else %}
|
||||||
{% if playlist_info.playlist_active %}
|
<button type="button" data-type="playlist" data-subscribe="true" data-id="{{ playlist_info.playlist_id }}" onclick="subscribeStatus(this)" title="Subscribe to {{ playlist_info.playlist_name }}">Subscribe</button>
|
||||||
<p>Youtube: <a href="https://www.youtube.com/playlist?list={{ playlist_info.playlist_id }}" target="_blank">Active</a></p>
|
{% endif %}
|
||||||
{% else %}
|
</p>
|
||||||
<p>Youtube: Deactivated</p>
|
{% if playlist_info.playlist_active %}
|
||||||
|
<p>Youtube: <a href="https://www.youtube.com/playlist?list={{ playlist_info.playlist_id }}" target="_blank">Active</a></p>
|
||||||
|
{% else %}
|
||||||
|
<p>Youtube: Deactivated</p>
|
||||||
|
{% endif %}
|
||||||
{% endif %}
|
{% endif %}
|
||||||
<button onclick="deleteConfirm()" id="delete-item">Delete Playlist</button>
|
<button onclick="deleteConfirm()" id="delete-item">Delete Playlist</button>
|
||||||
<div class="delete-confirm" id="delete-button">
|
<div class="delete-confirm" id="delete-button">
|
||||||
<span>Delete {{ playlist_info.playlist_name }}?</span>
|
<span>Delete {{ playlist_info.playlist_name }}?</span>
|
||||||
<button onclick="deletePlaylist(this)" data-action="metadata" data-id="{{ playlist_info.playlist_id }}">Delete metadata</button>
|
<button onclick="deletePlaylist(this)" data-action="" data-id="{{ playlist_info.playlist_id }}">Delete metadata</button>
|
||||||
<button onclick="deletePlaylist(this)" data-action="all" class="danger-button" data-id="{{ playlist_info.playlist_id }}">Delete all</button><br>
|
<button onclick="deletePlaylist(this)" data-action="delete-videos" class="danger-button" data-id="{{ playlist_info.playlist_id }}">Delete all</button><br>
|
||||||
<button onclick="cancelDelete()">Cancel</button>
|
<button onclick="cancelDelete()">Cancel</button>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
@ -50,13 +59,18 @@
|
||||||
<div>
|
<div>
|
||||||
{% if max_hits %}
|
{% if max_hits %}
|
||||||
<p>Total Videos archived: {{ max_hits }}/{{ playlist_info.playlist_entries|length }}</p>
|
<p>Total Videos archived: {{ max_hits }}/{{ playlist_info.playlist_entries|length }}</p>
|
||||||
<p>Watched: <button title="Mark all videos from {{ playlist_info.playlist_name }} as watched" type="button" id="watched-button" data-id="{{ playlist_info.playlist_id }}" onclick="isWatchedButton(this)">Mark as watched</button></p>
|
<div id="watched-button" class="button-box">
|
||||||
|
<button title="Mark all videos from {{ playlist_info.playlist_name }} as watched" type="button" id="watched-button" data-id="{{ playlist_info.playlist_id }}" onclick="isWatchedButton(this)">Mark as watched</button>
|
||||||
|
<button title="Mark all videos from {{ playlist_info.playlist_name }} as unwatched" type="button" id="unwatched-button" data-id="{{ playlist_info.playlist_id }}" onclick="isUnwatchedButton(this)">Mark as unwatched</button>
|
||||||
|
</div>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
{% if reindex %}
|
{% if reindex %}
|
||||||
<p>Reindex scheduled</p>
|
<p>Reindex scheduled</p>
|
||||||
{% else %}
|
{% else %}
|
||||||
<div id="reindex-button" class="button-box">
|
<div id="reindex-button" class="button-box">
|
||||||
<button data-id="{{ playlist_info.playlist_id }}" data-type="playlist" onclick="reindex(this)" title="Reindex Playlist {{ playlist_info.playlist_name }}">Reindex</button>
|
{% if playlist_info.playlist_type != "custom" %}
|
||||||
|
<button data-id="{{ playlist_info.playlist_id }}" data-type="playlist" onclick="reindex(this)" title="Reindex Playlist {{ playlist_info.playlist_name }}">Reindex</button>
|
||||||
|
{% endif %}
|
||||||
<button data-id="{{ playlist_info.playlist_id }}" data-type="playlist" data-extract-videos="true" onclick="reindex(this)" title="Reindex Videos of {{ playlist_info.playlist_name }}">Reindex Videos</button>
|
<button data-id="{{ playlist_info.playlist_id }}" data-type="playlist" data-extract-videos="true" onclick="reindex(this)" title="Reindex Videos of {{ playlist_info.playlist_name }}">Reindex Videos</button>
|
||||||
</div>
|
</div>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
@ -107,14 +121,14 @@
|
||||||
{% if results %}
|
{% if results %}
|
||||||
{% for video in results %}
|
{% for video in results %}
|
||||||
<div class="video-item {{ view_style }}">
|
<div class="video-item {{ view_style }}">
|
||||||
<a href="#player" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
|
<a href="#player" data-id="{{ video.youtube_id }}" onclick="createPlayer(this)">
|
||||||
<div class="video-thumb-wrap {{ view_style }}">
|
<div class="video-thumb-wrap {{ view_style }}">
|
||||||
<div class="video-thumb">
|
<div class="video-thumb">
|
||||||
<img src="{{ video.source.vid_thumb_url }}" alt="video-thumb">
|
<img src="{{ video.vid_thumb_url }}" alt="video-thumb">
|
||||||
{% if video.source.player.progress %}
|
{% if video.player.progress %}
|
||||||
<div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
|
<div class="video-progress-bar" id="progress-{{ video.youtube_id }}" style="width: {{video.player.progress}}%;"></div>
|
||||||
{% else %}
|
{% else %}
|
||||||
<div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: 0%;"></div>
|
<div class="video-progress-bar" id="progress-{{ video.youtube_id }}" style="width: 0%;"></div>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
<div class="video-play">
|
<div class="video-play">
|
||||||
|
@ -123,23 +137,43 @@
|
||||||
</div>
|
</div>
|
||||||
</a>
|
</a>
|
||||||
<div class="video-desc {{ view_style }}">
|
<div class="video-desc {{ view_style }}">
|
||||||
<div class="video-desc-player" id="video-info-{{ video.source.youtube_id }}">
|
<div class="video-desc-player" id="video-info-{{ video.youtube_id }}">
|
||||||
{% if video.source.player.watched %}
|
{% if video.player.watched %}
|
||||||
<img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" data-id="{{ video.source.youtube_id }}" data-status="watched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as unwatched">
|
<img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" data-id="{{ video.youtube_id }}" data-status="watched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as unwatched">
|
||||||
{% else %}
|
{% else %}
|
||||||
<img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" data-id="{{ video.source.youtube_id }}" data-status="unwatched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as watched">
|
<img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" data-id="{{ video.youtube_id }}" data-status="unwatched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as watched">
|
||||||
{% endif %}
|
{% endif %}
|
||||||
<span>{{ video.source.published }} | {{ video.source.player.duration_str }}</span>
|
<span>{{ video.published }} | {{ video.player.duration_str }}</span>
|
||||||
</div>
|
</div>
|
||||||
<div>
|
<div class="video-desc-details">
|
||||||
<a class="video-more" href="{% url 'video' video.source.youtube_id %}"><h2>{{ video.source.title }}</h2></a>
|
<div>
|
||||||
|
{% if playlist_info.playlist_type == "custom" %}
|
||||||
|
<a href="{% url 'channel_id' video.channel.channel_id %}"><h3>{{ video.channel.channel_name }}</h3></a>
|
||||||
|
{% endif %}
|
||||||
|
<a class="video-more" href="{% url 'video' video.youtube_id %}"><h2>{{ video.title }}</h2></a>
|
||||||
|
</div>
|
||||||
|
{% if playlist_info.playlist_type == "custom" %}
|
||||||
|
{% if pagination %}
|
||||||
|
{% if pagination.last_page > 0 %}
|
||||||
|
<img id="{{ video.youtube_id }}-button" src="{% static 'img/icon-dot-menu.svg' %}" alt="dot-menu-icon" data-id="{{ video.youtube_id }}" data-context="video" onclick="showCustomPlaylistMenu(this,'{{playlist_info.playlist_id}}',{{pagination.current_page}},{{pagination.last_page}})" class="dot-button" title="More actions">
|
||||||
|
{% else %}
|
||||||
|
<img id="{{ video.youtube_id }}-button" src="{% static 'img/icon-dot-menu.svg' %}" alt="dot-menu-icon" data-id="{{ video.youtube_id }}" data-context="video" onclick="showCustomPlaylistMenu(this,'{{playlist_info.playlist_id}}',{{pagination.current_page}},{{pagination.current_page}})" class="dot-button" title="More actions">
|
||||||
|
{% endif %}
|
||||||
|
{% else %}
|
||||||
|
<img id="{{ video.youtube_id }}-button" src="{% static 'img/icon-dot-menu.svg' %}" alt="dot-menu-icon" data-id="{{ video.youtube_id }}" data-context="video" onclick="showCustomPlaylistMenu(this,'{{playlist_info.playlist_id}}',0,0)" class="dot-button" title="More actions">
|
||||||
|
{% endif %}
|
||||||
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
{% else %}
|
{% else %}
|
||||||
<h2>No videos found...</h2>
|
<h2>No videos found...</h2>
|
||||||
<p>Try going to the <a href="{% url 'downloads' %}">downloads page</a> to start the scan and download tasks.</p>
|
{% if playlist_info.playlist_type == "custom" %}
|
||||||
|
<p>Try going to the <a href="{% url 'home' %}">home page</a> to add videos to this playlist.</p>
|
||||||
|
{% else %}
|
||||||
|
<p>Try going to the <a href="{% url 'downloads' %}">downloads page</a> to start the scan and download tasks.</p>
|
||||||
|
{% endif %}
|
||||||
{% endif %}
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|