API endpoints, #build

Changed:
- [API] Added delete video endpoint
- [API] Added delete channel endpoint
- [API] Added watched state endpoint
- Changed download queue interaction to existing endpoints
- Added update notification task
This commit is contained in:
simon 2022-12-23 23:01:28 +07:00
commit 14e0429758
No known key found for this signature in database
GPG Key ID: 2C15AA5E89985DD4
19 changed files with 643 additions and 315 deletions

348
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -41,6 +41,7 @@ Note:
- [Refresh](#refresh-view) - [Refresh](#refresh-view)
- [Cookie](#cookie-view) - [Cookie](#cookie-view)
- [Search](#search-view) - [Search](#search-view)
- [Watched](#watched-view)
- [Ping](#ping-view) - [Ping](#ping-view)
## Authentication ## Authentication
@ -79,7 +80,8 @@ Pass page number as a query parameter: `page=2`. Defaults to *0*, `page=1` is re
/api/video/ /api/video/
## Video Item View ## Video Item View
/api/video/\<video_id>/ GET: /api/video/\<video_id>/
DELETE: /api/video/\<video_id>/
## Video Comment View ## Video Comment View
/api/video/\<video_id>/comment/ /api/video/\<video_id>/comment/
@ -88,12 +90,12 @@ Pass page number as a query parameter: `page=2`. Defaults to *0*, `page=1` is re
/api/video/\<video_id>/similar/ /api/video/\<video_id>/similar/
## Video Progress View ## Video Progress View
/api/video/\<video_id>/progress /api/video/\<video_id>/progress/
Progress is stored for each user. Progress is stored for each user.
### Get last player position of a video ### Get last player position of a video
GET /api/video/\<video_id>/progress GET /api/video/\<video_id>/progress/
```json ```json
{ {
"youtube_id": "<video_id>", "youtube_id": "<video_id>",
@ -103,7 +105,7 @@ GET /api/video/\<video_id>/progress
``` ```
### Post player position of video ### Post player position of video
POST /api/video/\<video_id>/progress POST /api/video/\<video_id>/progress/
```json ```json
{ {
"position": 100 "position": 100
@ -111,7 +113,7 @@ POST /api/video/\<video_id>/progress
``` ```
### Delete player position of video ### Delete player position of video
DELETE /api/video/\<video_id>/progress DELETE /api/video/\<video_id>/progress/
## Sponsor Block View ## Sponsor Block View
@ -164,7 +166,9 @@ POST /api/channel/
``` ```
## Channel Item View ## Channel Item View
/api/channel/\<channel_id>/ GET: /api/channel/\<channel_id>/
DELETE: /api/channel/\<channel_id>/
- Will delete channel with all it's videos
## Channel Videos View ## Channel Videos View
/api/channel/\<channel_id>/video/ /api/channel/\<channel_id>/video/
@ -264,7 +268,7 @@ Remove this snapshot from index
## Login View ## Login View
Return token and user ID for username and password: Return token and user ID for username and password:
POST /api/login POST /api/login/
```json ```json
{ {
"username": "tubearchivist", "username": "tubearchivist",
@ -401,9 +405,22 @@ GET /api/search/?query=\<query>
Returns search results from your query. Returns search results from your query.
## Watched View
POST /api/watched/
Change watched state, where the `id` can be a single video, or channel/playlist to change all videos belonging to that channel/playlist.
```json
{
"id": "xxxxxxx",
"is_watched": True
}
```
## Ping View ## Ping View
Validate your connection with the API Validate your connection with the API
GET /api/ping GET /api/ping/
When valid returns message with user id: When valid returns message with user id:
```json ```json

View File

@ -23,6 +23,7 @@ from api.views import (
VideoProgressView, VideoProgressView,
VideoSimilarView, VideoSimilarView,
VideoSponsorView, VideoSponsorView,
WatchedView,
) )
from django.urls import path from django.urls import path
@ -124,6 +125,11 @@ urlpatterns = [
CookieView.as_view(), CookieView.as_view(),
name="api-cookie", name="api-cookie",
), ),
path(
"watched/",
WatchedView.as_view(),
name="api-watched",
),
path( path(
"search/", "search/",
SearchView.as_view(), SearchView.as_view(),

View File

@ -7,9 +7,11 @@ from home.src.download.yt_dlp_base import CookieHandler
from home.src.es.connect import ElasticWrap from home.src.es.connect import ElasticWrap
from home.src.es.snapshot import ElasticSnapshot from home.src.es.snapshot import ElasticSnapshot
from home.src.frontend.searching import SearchForm from home.src.frontend.searching import SearchForm
from home.src.frontend.watched import WatchState
from home.src.index.channel import YoutubeChannel
from home.src.index.generic import Pagination from home.src.index.generic import Pagination
from home.src.index.reindex import ReindexProgress from home.src.index.reindex import ReindexProgress
from home.src.index.video import SponsorBlock from home.src.index.video import SponsorBlock, YoutubeVideo
from home.src.ta.config import AppConfig from home.src.ta.config import AppConfig
from home.src.ta.helper import UrlListParser from home.src.ta.helper import UrlListParser
from home.src.ta.ta_redis import RedisArchivist, RedisQueue from home.src.ta.ta_redis import RedisArchivist, RedisQueue
@ -95,6 +97,20 @@ class VideoApiView(ApiBaseView):
self.get_document(video_id) self.get_document(video_id)
return Response(self.response, status=self.status_code) return Response(self.response, status=self.status_code)
def delete(self, request, video_id):
# pylint: disable=unused-argument
"""delete single video"""
message = {"video": video_id}
try:
YoutubeVideo(video_id).delete_media_file()
status_code = 200
message.update({"state": "delete"})
except FileNotFoundError:
status_code = 404
message.update({"state": "not found"})
return Response(message, status=status_code)
class VideoApiListView(ApiBaseView): class VideoApiListView(ApiBaseView):
"""resolves to /api/video/ """resolves to /api/video/
@ -251,6 +267,20 @@ class ChannelApiView(ApiBaseView):
self.get_document(channel_id) self.get_document(channel_id)
return Response(self.response, status=self.status_code) return Response(self.response, status=self.status_code)
def delete(self, request, channel_id):
# pylint: disable=unused-argument
"""delete channel"""
message = {"channel": channel_id}
try:
YoutubeChannel(channel_id).delete_channel()
status_code = 200
message.update({"state": "delete"})
except FileNotFoundError:
status_code = 404
message.update({"state": "not found"})
return Response(message, status=status_code)
class ChannelApiListView(ApiBaseView): class ChannelApiListView(ApiBaseView):
"""resolves to /api/channel/ """resolves to /api/channel/
@ -375,7 +405,7 @@ class DownloadApiView(ApiBaseView):
def post(self, request, video_id): def post(self, request, video_id):
"""post to video to change status""" """post to video to change status"""
item_status = request.data["status"] item_status = request.data.get("status")
if item_status not in self.valid_status: if item_status not in self.valid_status:
message = f"{video_id}: invalid status {item_status}" message = f"{video_id}: invalid status {item_status}"
print(message) print(message)
@ -674,6 +704,24 @@ class CookieView(ApiBaseView):
return Response(message) return Response(message)
class WatchedView(ApiBaseView):
"""resolves to /api/watched/
POST: change watched state of video, channel or playlist
"""
def post(self, request):
"""change watched state"""
youtube_id = request.data.get("id")
is_watched = request.data.get("is_watched")
if not youtube_id or is_watched is None:
message = {"message": "missing id or is_watched"}
return Response(message, status=400)
WatchState(youtube_id, is_watched).change()
return Response({"message": "success"}, status=200)
class SearchView(ApiBaseView): class SearchView(ApiBaseView):
"""resolves to /api/search/ """resolves to /api/search/
GET: run a search with the string in the ?query parameter GET: run a search with the string in the ?query parameter

View File

@ -262,4 +262,4 @@ CORS_ALLOW_HEADERS = list(default_headers) + [
# TA application settings # TA application settings
TA_UPSTREAM = "https://github.com/tubearchivist/tubearchivist" TA_UPSTREAM = "https://github.com/tubearchivist/tubearchivist"
TA_VERSION = "v0.3.0" TA_VERSION = "v0.3.1-unstable"

View File

@ -8,6 +8,7 @@ from home.src.es.connect import ElasticWrap
from home.src.es.index_setup import ElasitIndexWrap from home.src.es.index_setup import ElasitIndexWrap
from home.src.es.snapshot import ElasticSnapshot from home.src.es.snapshot import ElasticSnapshot
from home.src.ta.config import AppConfig as ArchivistConfig from home.src.ta.config import AppConfig as ArchivistConfig
from home.src.ta.config import ReleaseVersion
from home.src.ta.helper import clear_dl_cache from home.src.ta.helper import clear_dl_cache
from home.src.ta.ta_redis import RedisArchivist from home.src.ta.ta_redis import RedisArchivist
@ -34,6 +35,7 @@ class StartupCheck:
self.make_folders() self.make_folders()
clear_dl_cache(self.config_handler.config) clear_dl_cache(self.config_handler.config)
self.snapshot_check() self.snapshot_check()
self.ta_version_check()
self.set_has_run() self.set_has_run()
def get_has_run(self): def get_has_run(self):
@ -120,6 +122,10 @@ class StartupCheck:
print("elasticsearch version check passed") print("elasticsearch version check passed")
def ta_version_check(self):
"""remove key if updated now"""
ReleaseVersion().is_updated()
class HomeConfig(AppConfig): class HomeConfig(AppConfig):
"""call startup funcs""" """call startup funcs"""

View File

@ -49,6 +49,7 @@
"check_reindex_days": 90, "check_reindex_days": 90,
"thumbnail_check": {"minute": "0", "hour": "17", "day_of_week": "*"}, "thumbnail_check": {"minute": "0", "hour": "17", "day_of_week": "*"},
"run_backup": {"minute": "0", "hour": "8", "day_of_week": "0"}, "run_backup": {"minute": "0", "hour": "8", "day_of_week": "0"},
"run_backup_rotate": 5 "run_backup_rotate": 5,
"version_check": {"minute": "0", "hour": "11", "day_of_week": "*"}
} }
} }

View File

@ -15,7 +15,7 @@ from home.src.download.subscriptions import PlaylistSubscription
from home.src.download.yt_dlp_base import CookieHandler, YtWrap from home.src.download.yt_dlp_base import CookieHandler, YtWrap
from home.src.es.connect import ElasticWrap, IndexPaginate from home.src.es.connect import ElasticWrap, IndexPaginate
from home.src.index.channel import YoutubeChannel from home.src.index.channel import YoutubeChannel
from home.src.index.comments import Comments from home.src.index.comments import CommentList
from home.src.index.playlist import YoutubePlaylist from home.src.index.playlist import YoutubePlaylist
from home.src.index.video import YoutubeVideo, index_new_video from home.src.index.video import YoutubeVideo, index_new_video
from home.src.ta.config import AppConfig from home.src.ta.config import AppConfig
@ -143,25 +143,7 @@ class DownloadPostProcess:
def get_comments(self): def get_comments(self):
"""get comments from youtube""" """get comments from youtube"""
if not self.download.config["downloads"]["comment_max"]: CommentList(self.download.videos).index(notify=True)
return
total_videos = len(self.download.videos)
for idx, video_id in enumerate(self.download.videos):
comment = Comments(video_id, config=self.download.config)
comment.build_json(notify=(idx, total_videos))
if comment.json_data:
comment.upload_comments()
key = "message:download"
message = {
"status": key,
"level": "info",
"title": "Download and index comments finished",
"message": f"added comments for {total_videos} videos",
}
RedisArchivist().set_message(key, message, expire=4)
class VideoDownloader: class VideoDownloader:

View File

@ -4,15 +4,11 @@ Functionality:
- called via user input - called via user input
""" """
from home.src.download.queue import PendingInteract
from home.src.download.subscriptions import ( from home.src.download.subscriptions import (
ChannelSubscription, ChannelSubscription,
PlaylistSubscription, PlaylistSubscription,
) )
from home.src.frontend.watched import WatchState
from home.src.index.channel import YoutubeChannel
from home.src.index.playlist import YoutubePlaylist from home.src.index.playlist import YoutubePlaylist
from home.src.index.video import YoutubeVideo
from home.src.ta.helper import UrlListParser from home.src.ta.helper import UrlListParser
from home.src.ta.ta_redis import RedisArchivist, RedisQueue from home.src.ta.ta_redis import RedisArchivist, RedisQueue
from home.tasks import ( from home.tasks import (
@ -50,12 +46,9 @@ class PostData:
def exec_map(self): def exec_map(self):
"""map dict key and return function to execute""" """map dict key and return function to execute"""
exec_map = { exec_map = {
"watched": self._watched,
"un_watched": self._un_watched,
"change_view": self._change_view, "change_view": self._change_view,
"change_grid": self._change_grid, "change_grid": self._change_grid,
"rescan_pending": self._rescan_pending, "rescan_pending": self._rescan_pending,
"ignore": self._ignore,
"dl_pending": self._dl_pending, "dl_pending": self._dl_pending,
"queue": self._queue_handler, "queue": self._queue_handler,
"unsubscribe": self._unsubscribe, "unsubscribe": self._unsubscribe,
@ -65,32 +58,17 @@ class PostData:
"show_subed_only": self._show_subed_only, "show_subed_only": self._show_subed_only,
"dlnow": self._dlnow, "dlnow": self._dlnow,
"show_ignored_only": self._show_ignored_only, "show_ignored_only": self._show_ignored_only,
"forgetIgnore": self._forget_ignore,
"addSingle": self._add_single,
"deleteQueue": self._delete_queue,
"manual-import": self._manual_import, "manual-import": self._manual_import,
"re-embed": self._re_embed, "re-embed": self._re_embed,
"db-backup": self._db_backup, "db-backup": self._db_backup,
"db-restore": self._db_restore, "db-restore": self._db_restore,
"fs-rescan": self._fs_rescan, "fs-rescan": self._fs_rescan,
"delete-video": self._delete_video,
"delete-channel": self._delete_channel,
"delete-playlist": self._delete_playlist, "delete-playlist": self._delete_playlist,
"find-playlists": self._find_playlists, "find-playlists": self._find_playlists,
} }
return exec_map[self.to_exec] return exec_map[self.to_exec]
def _watched(self):
"""mark as watched"""
WatchState(self.exec_val).mark_as_watched()
return {"success": True}
def _un_watched(self):
"""mark as unwatched"""
WatchState(self.exec_val).mark_as_unwatched()
return {"success": True}
def _change_view(self): def _change_view(self):
"""process view changes in home, channel, and downloads""" """process view changes in home, channel, and downloads"""
origin, new_view = self.exec_val.split(":") origin, new_view = self.exec_val.split(":")
@ -117,15 +95,6 @@ class PostData:
update_subscribed.delay() update_subscribed.delay()
return {"success": True} return {"success": True}
def _ignore(self):
"""ignore from download queue"""
video_id = self.exec_val
print(f"{video_id}: ignore video from download queue")
PendingInteract(video_id=video_id, status="ignore").update_status()
# also clear from redis queue
RedisQueue(queue_name="dl_queue").clear_item(video_id)
return {"success": True}
@staticmethod @staticmethod
def _dl_pending(): def _dl_pending():
"""start the download queue""" """start the download queue"""
@ -228,27 +197,6 @@ class PostData:
RedisArchivist().set_message(key, value) RedisArchivist().set_message(key, value)
return {"success": True} return {"success": True}
def _forget_ignore(self):
"""delete from ta_download index"""
video_id = self.exec_val
print(f"{video_id}: forget from download")
PendingInteract(video_id=video_id).delete_item()
return {"success": True}
def _add_single(self):
"""add single youtube_id to download queue"""
video_id = self.exec_val
print(f"{video_id}: add single vid to download queue")
PendingInteract(video_id=video_id, status="pending").update_status()
return {"success": True}
def _delete_queue(self):
"""delete download queue"""
status = self.exec_val
print("deleting from download queue: " + status)
PendingInteract(status=status).delete_by_status()
return {"success": True}
@staticmethod @staticmethod
def _manual_import(): def _manual_import():
"""run manual import from settings page""" """run manual import from settings page"""
@ -284,18 +232,6 @@ class PostData:
rescan_filesystem.delay() rescan_filesystem.delay()
return {"success": True} return {"success": True}
def _delete_video(self):
"""delete media file, metadata and thumb"""
youtube_id = self.exec_val
YoutubeVideo(youtube_id).delete_media_file()
return {"success": True}
def _delete_channel(self):
"""delete channel and all matching videos"""
channel_id = self.exec_val
YoutubeChannel(channel_id).delete_channel()
return {"success": True}
def _delete_playlist(self): def _delete_playlist(self):
"""delete playlist, only metadata or incl all videos""" """delete playlist, only metadata or incl all videos"""
playlist_dict = self.exec_val playlist_dict = self.exec_val

View File

@ -12,95 +12,94 @@ from home.src.ta.helper import UrlListParser
class WatchState: class WatchState:
"""handle watched checkbox for videos and channels""" """handle watched checkbox for videos and channels"""
def __init__(self, youtube_id): def __init__(self, youtube_id, is_watched):
self.youtube_id = youtube_id self.youtube_id = youtube_id
self.is_watched = is_watched
self.stamp = int(datetime.now().timestamp()) self.stamp = int(datetime.now().timestamp())
self.pipeline = f"_ingest/pipeline/watch_{youtube_id}"
def mark_as_watched(self): def change(self):
"""update es with new watched value""" """change watched state of item(s)"""
url_type = self.dedect_type() url_type = self._dedect_type()
if url_type == "video": if url_type == "video":
self.mark_vid_watched() self.change_vid_state()
elif url_type == "channel": return
self.mark_channel_watched()
elif url_type == "playlist":
self.mark_playlist_watched()
print(f"{self.youtube_id}: marked as watched") self._add_pipeline()
path = f"ta_video/_update_by_query?pipeline=watch_{self.youtube_id}"
data = self._build_update_data(url_type)
_, _ = ElasticWrap(path).post(data)
self._delete_pipeline()
def mark_as_unwatched(self): def _dedect_type(self):
"""revert watched state to false"""
url_type = self.dedect_type()
if url_type == "video":
self.mark_vid_watched(revert=True)
print(f"{self.youtube_id}: revert as unwatched")
def dedect_type(self):
"""find youtube id type""" """find youtube id type"""
print(self.youtube_id) print(self.youtube_id)
url_process = UrlListParser(self.youtube_id).process_list() url_process = UrlListParser(self.youtube_id).process_list()
url_type = url_process[0]["type"] url_type = url_process[0]["type"]
return url_type return url_type
def mark_vid_watched(self, revert=False): def change_vid_state(self):
"""change watched status of single video""" """change watched state of video"""
path = f"ta_video/_update/{self.youtube_id}" path = f"ta_video/_update/{self.youtube_id}"
data = { data = {
"doc": {"player": {"watched": True, "watched_date": self.stamp}} "doc": {
"player": {
"watched": self.is_watched,
"watched_date": self.stamp,
}
}
} }
if revert:
data["doc"]["player"]["watched"] = False
response, status_code = ElasticWrap(path).post(data=data) response, status_code = ElasticWrap(path).post(data=data)
if status_code != 200: if status_code != 200:
print(response) print(response)
raise ValueError("failed to mark video as watched") raise ValueError("failed to mark video as watched")
def _get_source(self): def _build_update_data(self, url_type):
"""build source line for update_by_query script""" """build update by query data based on url_type"""
source = [ term_key_map = {
"ctx._source.player['watched'] = true", "channel": "channel.channel_id",
f"ctx._source.player['watched_date'] = {self.stamp}", "playlist": "playlist.keyword",
] }
return "; ".join(source) term_key = term_key_map.get(url_type)
def mark_channel_watched(self): return {
"""change watched status of every video in channel""" "query": {
path = "ta_video/_update_by_query" "bool": {
must_list = [ "must": [
{"term": {"channel.channel_id": {"value": self.youtube_id}}}, {"term": {term_key: {"value": self.youtube_id}}},
{"term": {"player.watched": {"value": False}}}, {
] "term": {
data = { "player.watched": {
"query": {"bool": {"must": must_list}}, "value": not self.is_watched
"script": { }
"source": self._get_source(), }
"lang": "painless",
}, },
],
}
}
} }
response, status_code = ElasticWrap(path).post(data=data) def _add_pipeline(self):
if status_code != 200: """add ingest pipeline"""
print(response)
raise ValueError("failed mark channel as watched")
def mark_playlist_watched(self):
"""change watched state of all videos in playlist"""
path = "ta_video/_update_by_query"
must_list = [
{"term": {"playlist.keyword": {"value": self.youtube_id}}},
{"term": {"player.watched": {"value": False}}},
]
data = { data = {
"query": {"bool": {"must": must_list}}, "description": f"{self.youtube_id}: watched {self.is_watched}",
"script": { "processors": [
"source": self._get_source(), {
"lang": "painless", "set": {
}, "field": "player.watched",
"value": self.is_watched,
} }
},
{
"set": {
"field": "player.watched_date",
"value": self.stamp,
}
},
],
}
_, _ = ElasticWrap(self.pipeline).put(data)
response, status_code = ElasticWrap(path).post(data=data) def _delete_pipeline(self):
if status_code != 200: """delete pipeline"""
print(response) ElasticWrap(self.pipeline).delete()
raise ValueError("failed mark playlist as watched")

View File

@ -298,6 +298,9 @@ class YoutubeChannel(YouTubeItem):
"""delete channel and all videos""" """delete channel and all videos"""
print(f"{self.youtube_id}: delete channel") print(f"{self.youtube_id}: delete channel")
self.get_from_es() self.get_from_es()
if not self.json_data:
raise FileNotFoundError
folder_path = self.get_folder_path() folder_path = self.get_folder_path()
print(f"{self.youtube_id}: delete all media files") print(f"{self.youtube_id}: delete all media files")
try: try:

View File

@ -14,7 +14,7 @@ from home.src.ta.ta_redis import RedisArchivist
class Comments: class Comments:
"""hold all comments functionality""" """interact with comments per video"""
def __init__(self, youtube_id, config=False): def __init__(self, youtube_id, config=False):
self.youtube_id = youtube_id self.youtube_id = youtube_id
@ -146,6 +146,7 @@ class Comments:
if not self.is_activated: if not self.is_activated:
return return
print(f"{self.youtube_id}: upload comments")
_, _ = ElasticWrap(self.es_path).put(self.json_data) _, _ = ElasticWrap(self.es_path).put(self.json_data)
vid_path = f"ta_video/_update/{self.youtube_id}" vid_path = f"ta_video/_update/{self.youtube_id}"
@ -187,3 +188,40 @@ class Comments:
self.delete_comments() self.delete_comments()
self.upload_comments() self.upload_comments()
class CommentList:
"""interact with comments in group"""
def __init__(self, video_ids):
self.video_ids = video_ids
self.config = AppConfig().config
def index(self, notify=False):
"""index group of videos"""
if not self.config["downloads"].get("comment_max"):
return
total_videos = len(self.video_ids)
for idx, video_id in enumerate(self.video_ids):
comment = Comments(video_id, config=self.config)
if notify:
notify = (idx, total_videos)
comment.build_json(notify=notify)
if comment.json_data:
comment.upload_comments()
if notify:
self.notify_final(total_videos)
@staticmethod
def notify_final(total_videos):
"""send final notification"""
key = "message:download"
message = {
"status": key,
"level": "info",
"title": "Download and index comments finished",
"message": f"added comments for {total_videos} videos",
}
RedisArchivist().set_message(key, message, expire=4)

View File

@ -14,6 +14,7 @@ import subprocess
from home.src.download.queue import PendingList from home.src.download.queue import PendingList
from home.src.download.thumbnails import ThumbManager from home.src.download.thumbnails import ThumbManager
from home.src.es.connect import ElasticWrap from home.src.es.connect import ElasticWrap
from home.src.index.comments import CommentList
from home.src.index.video import YoutubeVideo, index_new_video from home.src.index.video import YoutubeVideo, index_new_video
from home.src.ta.config import AppConfig from home.src.ta.config import AppConfig
from home.src.ta.helper import clean_string, ignore_filelist from home.src.ta.helper import clean_string, ignore_filelist
@ -601,6 +602,8 @@ def scan_filesystem():
filesystem_handler.delete_from_index() filesystem_handler.delete_from_index()
if filesystem_handler.to_index: if filesystem_handler.to_index:
print("index new videos") print("index new videos")
for missing_vid in filesystem_handler.to_index: video_ids = [i[2] for i in filesystem_handler.to_index]
youtube_id = missing_vid[2] for youtube_id in video_ids:
index_new_video(youtube_id) index_new_video(youtube_id)
CommentList(video_ids).index()

View File

@ -292,6 +292,9 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
"""delete video file, meta data""" """delete video file, meta data"""
print(f"{self.youtube_id}: delete video") print(f"{self.youtube_id}: delete video")
self.get_from_es() self.get_from_es()
if not self.json_data:
raise FileNotFoundError
video_base = self.app_conf["videos"] video_base = self.app_conf["videos"]
media_url = self.json_data.get("media_url") media_url = self.json_data.get("media_url")
file_path = os.path.join(video_base, media_url) file_path = os.path.join(video_base, media_url)

View File

@ -8,7 +8,9 @@ import json
import os import os
import re import re
import requests
from celery.schedules import crontab from celery.schedules import crontab
from django.conf import settings
from home.src.ta.ta_redis import RedisArchivist from home.src.ta.ta_redis import RedisArchivist
@ -154,6 +156,7 @@ class ScheduleBuilder:
"check_reindex": "0 12 *", "check_reindex": "0 12 *",
"thumbnail_check": "0 17 *", "thumbnail_check": "0 17 *",
"run_backup": "0 18 0", "run_backup": "0 18 0",
"version_check": "0 11 *",
} }
CONFIG = ["check_reindex_days", "run_backup_rotate"] CONFIG = ["check_reindex_days", "run_backup_rotate"]
MSG = "message:setting" MSG = "message:setting"
@ -268,3 +271,74 @@ class ScheduleBuilder:
schedule_dict.update(to_add) schedule_dict.update(to_add)
return schedule_dict return schedule_dict
class ReleaseVersion:
"""compare local version with remote version"""
REMOTE_URL = "https://www.tubearchivist.com/api/release/latest/"
NEW_KEY = "versioncheck:new"
def __init__(self):
self.local_version = self._parse_version(settings.TA_VERSION)
self.is_unstable = settings.TA_VERSION.endswith("-unstable")
self.remote_version = False
self.is_breaking = False
self.response = False
def check(self):
"""check version"""
print(f"[{self.local_version}]: look for updates")
self.get_remote_version()
new_version, is_breaking = self._has_update()
if new_version:
message = {
"status": True,
"version": new_version,
"is_breaking": is_breaking,
}
RedisArchivist().set_message(self.NEW_KEY, message)
print(f"[{self.local_version}]: found new version {new_version}")
def get_remote_version(self):
"""read version from remote"""
self.response = requests.get(self.REMOTE_URL, timeout=20).json()
remote_version_str = self.response["release_version"]
self.remote_version = self._parse_version(remote_version_str)
self.is_breaking = self.response["breaking_changes"]
def _has_update(self):
"""check if there is an update"""
for idx, number in enumerate(self.local_version):
is_newer = self.remote_version[idx] > number
if is_newer:
return self.response["release_version"], self.is_breaking
if self.is_unstable and self.local_version == self.remote_version:
return self.response["release_version"], self.is_breaking
return False, False
@staticmethod
def _parse_version(version):
"""return version parts"""
clean = version.rstrip("-unstable").lstrip("v")
return tuple((int(i) for i in clean.split(".")))
def is_updated(self):
"""check if update happened in the mean time"""
message = self.get_update()
if not message:
return
if self._parse_version(message.get("version")) == self.local_version:
print(f"[{self.local_version}]: update completed")
RedisArchivist().del_message(self.NEW_KEY)
def get_update(self):
"""return new version dict if available"""
message = RedisArchivist().get_message(self.NEW_KEY)
if not message.get("status"):
return False
return message

View File

@ -22,7 +22,7 @@ from home.src.es.index_setup import ElasitIndexWrap
from home.src.index.channel import YoutubeChannel from home.src.index.channel import YoutubeChannel
from home.src.index.filesystem import ImportFolderScanner, scan_filesystem from home.src.index.filesystem import ImportFolderScanner, scan_filesystem
from home.src.index.reindex import Reindex, ReindexManual, ReindexOutdated from home.src.index.reindex import Reindex, ReindexManual, ReindexOutdated
from home.src.ta.config import AppConfig, ScheduleBuilder from home.src.ta.config import AppConfig, ReleaseVersion, ScheduleBuilder
from home.src.ta.helper import UrlListParser, clear_dl_cache from home.src.ta.helper import UrlListParser, clear_dl_cache
from home.src.ta.ta_redis import RedisArchivist, RedisQueue from home.src.ta.ta_redis import RedisArchivist, RedisQueue
@ -290,9 +290,15 @@ def index_channel_playlists(channel_id):
channel.index_channel_playlists() channel.index_channel_playlists()
@shared_task(name="version_check")
def version_check():
"""check for new updates"""
ReleaseVersion().check()
try: try:
app.conf.beat_schedule = ScheduleBuilder().build_schedule() app.conf.beat_schedule = ScheduleBuilder().build_schedule()
except KeyError: except KeyError:
# update path from v0.0.8 to v0.0.9 to load new defaults # update path to load new defaults
StartupCheck().sync_redis_state() StartupCheck().sync_redis_state()
app.conf.beat_schedule = ScheduleBuilder().build_schedule() app.conf.beat_schedule = ScheduleBuilder().build_schedule()

View File

@ -132,7 +132,14 @@
</div> </div>
<div class="footer"> <div class="footer">
<div class="boxed-content"> <div class="boxed-content">
<span>© 2021 - <script type="text/javascript">document.write(new Date().getFullYear());</script> TubeArchivist {{ version }} </span><span><a href="{% url 'about' %}">About</a> | <a href="https://github.com/tubearchivist/tubearchivist" target="_blank">GitHub</a> | <a href="https://hub.docker.com/r/bbilly1/tubearchivist" target="_blank">Docker Hub</a> | <a href="https://www.tubearchivist.com/discord" target="_blank">Discord</a> | <a href="https://www.reddit.com/r/TubeArchivist/">Reddit</a></span> <span>© 2021 - <script type="text/javascript">document.write(new Date().getFullYear());</script></span>
<span>TubeArchivist</span>
<span>{{ version }}</span>
{% if ta_update %}
<span class="danger-zone">{{ ta_update.version }} available{% if ta_update.is_breaking %}<span class="danger-zone">Breaking Changes!</span>{% endif %}</span>
<span><a href="https://github.com/tubearchivist/tubearchivist/releases/tag/{{ ta_update.version }}" target="_blank">Release Page</a> | </span>
{% endif %}
<span><a href="{% url 'about' %}">About</a> | <a href="https://github.com/tubearchivist/tubearchivist" target="_blank">GitHub</a> | <a href="https://hub.docker.com/r/bbilly1/tubearchivist" target="_blank">Docker Hub</a> | <a href="https://www.tubearchivist.com/discord" target="_blank">Discord</a> | <a href="https://www.reddit.com/r/TubeArchivist/">Reddit</a></span>
</div> </div>
</div> </div>
</body> </body>

View File

@ -36,7 +36,7 @@ from home.src.index.channel import YoutubeChannel, channel_overwrites
from home.src.index.generic import Pagination from home.src.index.generic import Pagination
from home.src.index.playlist import YoutubePlaylist from home.src.index.playlist import YoutubePlaylist
from home.src.index.reindex import ReindexProgress from home.src.index.reindex import ReindexProgress
from home.src.ta.config import AppConfig, ScheduleBuilder from home.src.ta.config import AppConfig, ReleaseVersion, ScheduleBuilder
from home.src.ta.helper import UrlListParser, time_parser from home.src.ta.helper import UrlListParser, time_parser
from home.src.ta.ta_redis import RedisArchivist from home.src.ta.ta_redis import RedisArchivist
from home.tasks import extrac_dl, index_channel_playlists, subscribe_to from home.tasks import extrac_dl, index_channel_playlists, subscribe_to
@ -138,6 +138,7 @@ class ArchivistViewConfig(View):
"show_ignored_only": self._get_show_ignore_only(), "show_ignored_only": self._get_show_ignore_only(),
"show_subed_only": self._get_show_subed_only(), "show_subed_only": self._get_show_subed_only(),
"version": settings.TA_VERSION, "version": settings.TA_VERSION,
"ta_update": ReleaseVersion().get_update(),
} }
@ -257,6 +258,19 @@ class ArchivistResultsView(ArchivistViewConfig):
self.context["pagination"] = self.pagination_handler.pagination self.context["pagination"] = self.pagination_handler.pagination
class MinView(View):
"""to inherit from for minimal config vars"""
@staticmethod
def get_min_context(request):
"""build minimal vars for context"""
return {
"colors": AppConfig(request.user.id).colors,
"version": settings.TA_VERSION,
"ta_update": ReleaseVersion().get_update(),
}
class HomeView(ArchivistResultsView): class HomeView(ArchivistResultsView):
"""resolves to / """resolves to /
handle home page and video search post functionality handle home page and video search post functionality
@ -298,20 +312,23 @@ class HomeView(ArchivistResultsView):
self.data["query"] = query self.data["query"] = query
class LoginView(View): class LoginView(MinView):
"""resolves to /login/ """resolves to /login/
Greeting and login page Greeting and login page
""" """
SEC_IN_DAY = 60 * 60 * 24 SEC_IN_DAY = 60 * 60 * 24
@staticmethod def get(self, request):
def get(request):
"""handle get requests""" """handle get requests"""
failed = bool(request.GET.get("failed")) context = self.get_min_context(request)
colors = AppConfig(request.user.id).colors context.update(
form = CustomAuthForm() {
context = {"colors": colors, "form": form, "form_error": failed} "form": CustomAuthForm(),
"form_error": bool(request.GET.get("failed")),
}
)
return render(request, "home/login.html", context) return render(request, "home/login.html", context)
def post(self, request): def post(self, request):
@ -333,19 +350,15 @@ class LoginView(View):
return redirect("/login?failed=true") return redirect("/login?failed=true")
class AboutView(View): class AboutView(MinView):
"""resolves to /about/ """resolves to /about/
show helpful how to information show helpful how to information
""" """
@staticmethod def get(self, request):
def get(request):
"""handle http get""" """handle http get"""
context = { context = self.get_min_context(request)
"title": "About", context.update({"title": "About"})
"colors": AppConfig(request.user.id).colors,
"version": settings.TA_VERSION,
}
return render(request, "home/about.html", context) return render(request, "home/about.html", context)
@ -843,7 +856,7 @@ class PlaylistView(ArchivistResultsView):
return redirect("playlist") return redirect("playlist")
class VideoView(View): class VideoView(MinView):
"""resolves to /video/<video-id>/ """resolves to /video/<video-id>/
display details about a single video display details about a single video
""" """
@ -869,17 +882,18 @@ class VideoView(View):
request_type="video", request_id=video_id request_type="video", request_id=video_id
).get_progress() ).get_progress()
context = { context = self.get_min_context(request)
context.update(
{
"video": video_data, "video": video_data,
"playlist_nav": playlist_nav, "playlist_nav": playlist_nav,
"title": video_data.get("title"), "title": video_data.get("title"),
"colors": config_handler.colors,
"cast": config_handler.config["application"]["enable_cast"], "cast": config_handler.config["application"]["enable_cast"],
"version": settings.TA_VERSION,
"config": config_handler.config, "config": config_handler.config,
"position": time_parser(request.GET.get("t")), "position": time_parser(request.GET.get("t")),
"reindex": reindex.get("state"), "reindex": reindex.get("state"),
} }
)
return render(request, "home/video.html", context) return render(request, "home/video.html", context)
@staticmethod @staticmethod
@ -936,7 +950,7 @@ class SearchView(ArchivistResultsView):
return render(request, "home/search.html", self.context) return render(request, "home/search.html", self.context)
class SettingsView(View): class SettingsView(MinView):
"""resolves to /settings/ """resolves to /settings/
handle the settings page, display current settings, handle the settings page, display current settings,
take post request from the form to update settings take post request from the form to update settings
@ -944,28 +958,19 @@ class SettingsView(View):
def get(self, request): def get(self, request):
"""read and display current settings""" """read and display current settings"""
config_handler = AppConfig(request.user.id) context = self.get_min_context(request)
colors = config_handler.colors context.update(
{
available_backups = ElasticBackup().get_all_backup_files()
user_form = UserSettingsForm()
app_form = ApplicationSettingsForm()
scheduler_form = SchedulerSettingsForm()
snapshots = ElasticSnapshot().get_snapshot_stats()
token = self.get_token(request)
context = {
"title": "Settings", "title": "Settings",
"config": config_handler.config, "config": AppConfig(request.user.id).config,
"api_token": token, "api_token": self.get_token(request),
"colors": colors, "available_backups": ElasticBackup().get_all_backup_files(),
"available_backups": available_backups, "user_form": UserSettingsForm(),
"user_form": user_form, "app_form": ApplicationSettingsForm(),
"app_form": app_form, "scheduler_form": SchedulerSettingsForm(),
"scheduler_form": scheduler_form, "snapshots": ElasticSnapshot().get_snapshot_stats(),
"snapshots": snapshots,
"version": settings.TA_VERSION,
} }
)
return render(request, "home/settings.html", context) return render(request, "home/settings.html", context)

View File

@ -23,15 +23,14 @@ function updateVideoWatchStatus(input1, videoCurrentWatchStatus) {
postVideoProgress(videoId, 0); // Reset video progress on watched/unwatched; postVideoProgress(videoId, 0); // Reset video progress on watched/unwatched;
removeProgressBar(videoId); removeProgressBar(videoId);
let watchStatusIndicator, payload; let watchStatusIndicator;
let apiEndpoint = '/api/watched/';
if (videoCurrentWatchStatus === 'watched') { if (videoCurrentWatchStatus === 'watched') {
watchStatusIndicator = createWatchStatusIndicator(videoId, 'unwatched'); watchStatusIndicator = createWatchStatusIndicator(videoId, 'unwatched');
payload = JSON.stringify({ un_watched: videoId }); apiRequest(apiEndpoint, 'POST', { id: videoId, is_watched: false });
sendPost(payload);
} else if (videoCurrentWatchStatus === 'unwatched') { } else if (videoCurrentWatchStatus === 'unwatched') {
watchStatusIndicator = createWatchStatusIndicator(videoId, 'watched'); watchStatusIndicator = createWatchStatusIndicator(videoId, 'watched');
payload = JSON.stringify({ watched: videoId }); apiRequest(apiEndpoint, 'POST', { id: videoId, is_watched: true });
sendPost(payload);
} }
let watchButtons = document.getElementsByClassName('watch-button'); let watchButtons = document.getElementsByClassName('watch-button');
@ -76,9 +75,10 @@ function removeProgressBar(videoId) {
function isWatchedButton(button) { function isWatchedButton(button) {
let youtube_id = button.getAttribute('data-id'); let youtube_id = button.getAttribute('data-id');
let payload = JSON.stringify({ watched: youtube_id }); let apiEndpoint = '/api/watched/';
let data = { id: youtube_id, is_watched: true };
button.remove(); button.remove();
sendPost(payload); apiRequest(apiEndpoint, 'POST', data);
setTimeout(function () { setTimeout(function () {
location.reload(); location.reload();
}, 1000); }, 1000);
@ -186,8 +186,8 @@ function dlPending() {
function toIgnore(button) { function toIgnore(button) {
let youtube_id = button.getAttribute('data-id'); let youtube_id = button.getAttribute('data-id');
let payload = JSON.stringify({ ignore: youtube_id }); let apiEndpoint = '/api/download/' + youtube_id + '/';
sendPost(payload); apiRequest(apiEndpoint, 'POST', { status: 'ignore' });
document.getElementById('dl-' + youtube_id).remove(); document.getElementById('dl-' + youtube_id).remove();
} }
@ -203,15 +203,15 @@ function downloadNow(button) {
function forgetIgnore(button) { function forgetIgnore(button) {
let youtube_id = button.getAttribute('data-id'); let youtube_id = button.getAttribute('data-id');
let payload = JSON.stringify({ forgetIgnore: youtube_id }); let apiEndpoint = '/api/download/' + youtube_id + '/';
sendPost(payload); apiRequest(apiEndpoint, 'DELETE');
document.getElementById('dl-' + youtube_id).remove(); document.getElementById('dl-' + youtube_id).remove();
} }
function addSingle(button) { function addSingle(button) {
let youtube_id = button.getAttribute('data-id'); let youtube_id = button.getAttribute('data-id');
let payload = JSON.stringify({ addSingle: youtube_id }); let apiEndpoint = '/api/download/' + youtube_id + '/';
sendPost(payload); apiRequest(apiEndpoint, 'POST', { status: 'pending' });
document.getElementById('dl-' + youtube_id).remove(); document.getElementById('dl-' + youtube_id).remove();
setTimeout(function () { setTimeout(function () {
checkMessages(); checkMessages();
@ -220,8 +220,8 @@ function addSingle(button) {
function deleteQueue(button) { function deleteQueue(button) {
let to_delete = button.getAttribute('data-id'); let to_delete = button.getAttribute('data-id');
let payload = JSON.stringify({ deleteQueue: to_delete }); let apiEndpoint = '/api/download/?filter=' + to_delete;
sendPost(payload); apiRequest(apiEndpoint, 'DELETE');
// clear button // clear button
let message = document.createElement('p'); let message = document.createElement('p');
message.innerText = 'deleting download queue: ' + to_delete; message.innerText = 'deleting download queue: ' + to_delete;
@ -334,8 +334,8 @@ function deleteConfirm() {
function deleteVideo(button) { function deleteVideo(button) {
let to_delete = button.getAttribute('data-id'); let to_delete = button.getAttribute('data-id');
let to_redirect = button.getAttribute('data-redirect'); let to_redirect = button.getAttribute('data-redirect');
let payload = JSON.stringify({ 'delete-video': to_delete }); let apiEndpoint = '/api/video/' + to_delete + '/';
sendPost(payload); apiRequest(apiEndpoint, 'DELETE');
setTimeout(function () { setTimeout(function () {
let redirect = '/channel/' + to_redirect; let redirect = '/channel/' + to_redirect;
window.location.replace(redirect); window.location.replace(redirect);
@ -344,8 +344,8 @@ function deleteVideo(button) {
function deleteChannel(button) { function deleteChannel(button) {
let to_delete = button.getAttribute('data-id'); let to_delete = button.getAttribute('data-id');
let payload = JSON.stringify({ 'delete-channel': to_delete }); let apiEndpoint = '/api/channel/' + to_delete + '/';
sendPost(payload); apiRequest(apiEndpoint, 'DELETE');
setTimeout(function () { setTimeout(function () {
window.location.replace('/channel/'); window.location.replace('/channel/');
}, 1000); }, 1000);