API extensions, #build

Changed:
- [API] Added endpoints for subscription toggle
- [API] Added endpoint for playlist delete
- Trigger bgsave when storing redis config
- Validate subscribe url Type, surface errors
- ignore eaDir folder
This commit is contained in:
Simon 2023-08-24 00:06:54 +07:00
commit 15794ebfc8
No known key found for this signature in database
GPG Key ID: 2C15AA5E89985DD4
17 changed files with 160 additions and 113 deletions

View File

@ -2,6 +2,10 @@
from api.src.search_processor import SearchProcess
from home.src.download.queue import PendingInteract
from home.src.download.subscriptions import (
ChannelSubscription,
PlaylistSubscription,
)
from home.src.download.yt_dlp_base import CookieHandler
from home.src.es.connect import ElasticWrap
from home.src.es.snapshot import ElasticSnapshot
@ -9,6 +13,7 @@ from home.src.frontend.searching import SearchForm
from home.src.frontend.watched import WatchState
from home.src.index.channel import YoutubeChannel
from home.src.index.generic import Pagination
from home.src.index.playlist import YoutubePlaylist
from home.src.index.reindex import ReindexProgress
from home.src.index.video import SponsorBlock, YoutubeVideo
from home.src.ta.config import AppConfig, ReleaseVersion
@ -318,9 +323,8 @@ class ChannelApiListView(ApiBaseView):
return Response(self.response)
@staticmethod
def post(request):
"""subscribe to list of channels"""
def post(self, request):
"""subscribe/unsubscribe to list of channels"""
data = request.data
try:
to_add = data["data"]
@ -329,12 +333,28 @@ class ChannelApiListView(ApiBaseView):
print(message)
return Response({"message": message}, status=400)
pending = [i["channel_id"] for i in to_add if i["channel_subscribed"]]
url_str = " ".join(pending)
subscribe_to.delay(url_str)
pending = []
for channel_item in to_add:
channel_id = channel_item["channel_id"]
if channel_item["channel_subscribed"]:
pending.append(channel_id)
else:
self._unsubscribe(channel_id)
if pending:
url_str = " ".join(pending)
subscribe_to.delay(url_str, expected_type="channel")
return Response(data)
@staticmethod
def _unsubscribe(channel_id: str):
"""unsubscribe"""
print(f"[{channel_id}] unsubscribe from channel")
ChannelSubscription().change_subscribe(
channel_id, channel_subscribed=False
)
class ChannelApiVideoView(ApiBaseView):
"""resolves to /api/channel/<channel-id>/video
@ -373,6 +393,38 @@ class PlaylistApiListView(ApiBaseView):
self.get_document_list(request)
return Response(self.response)
def post(self, request):
"""subscribe/unsubscribe to list of playlists"""
data = request.data
try:
to_add = data["data"]
except KeyError:
message = "missing expected data key"
print(message)
return Response({"message": message}, status=400)
pending = []
for playlist_item in to_add:
playlist_id = playlist_item["playlist_id"]
if playlist_item["playlist_subscribed"]:
pending.append(playlist_id)
else:
self._unsubscribe(playlist_id)
if pending:
url_str = " ".join(pending)
subscribe_to.delay(url_str, expected_type="playlist")
return Response(data)
@staticmethod
def _unsubscribe(playlist_id: str):
"""unsubscribe"""
print(f"[{playlist_id}] unsubscribe from playlist")
PlaylistSubscription().change_subscribe(
playlist_id, subscribe_status=False
)
class PlaylistApiView(ApiBaseView):
"""resolves to /api/playlist/<playlist_id>/
@ -387,6 +439,17 @@ class PlaylistApiView(ApiBaseView):
self.get_document(playlist_id)
return Response(self.response, status=self.status_code)
def delete(self, request, playlist_id):
"""delete playlist"""
print(f"{playlist_id}: delete playlist")
delete_videos = request.GET.get("delete-videos", False)
if delete_videos:
YoutubePlaylist(playlist_id).delete_videos_playlist()
else:
YoutubePlaylist(playlist_id).delete_metadata()
return Response({"success": True})
class PlaylistApiVideoView(ApiBaseView):
"""resolves to /api/playlist/<playlist_id>/video

View File

@ -332,7 +332,7 @@ class SubscriptionHandler:
self.task = task
self.to_subscribe = False
def subscribe(self):
def subscribe(self, expected_type=False):
"""subscribe to url_str items"""
if self.task:
self.task.send_progress(["Processing form content."])
@ -343,11 +343,16 @@ class SubscriptionHandler:
if self.task:
self._notify(idx, item, total)
self.subscribe_type(item)
self.subscribe_type(item, expected_type=expected_type)
def subscribe_type(self, item):
def subscribe_type(self, item, expected_type):
"""process single item"""
if item["type"] == "playlist":
if expected_type and expected_type != "playlist":
raise TypeError(
f"expected {expected_type} url but got {item.get('type')}"
)
PlaylistSubscription().process_url_str([item])
return
@ -360,6 +365,11 @@ class SubscriptionHandler:
else:
raise ValueError("failed to subscribe to: " + item["url"])
if expected_type and expected_type != "channel":
raise TypeError(
f"expected {expected_type} url but got {item.get('type')}"
)
self._subscribe(channel_id)
def _subscribe(self, channel_id):

View File

@ -61,7 +61,7 @@ class ThumbManagerBase:
print(f"{self.item_id}: retry thumbnail download {url}")
sleep((i + 1) ** i)
return False
return self.get_fallback()
def get_fallback(self):
"""get fallback thumbnail if not available"""

View File

@ -112,9 +112,9 @@ class CookieHandler:
def set_cookie(self, cookie):
"""set cookie str and activate in cofig"""
RedisArchivist().set_message("cookie", cookie)
RedisArchivist().set_message("cookie", cookie, save=True)
path = ".downloads.cookie_import"
RedisArchivist().set_message("config", True, path=path)
RedisArchivist().set_message("config", True, path=path, save=True)
self.config["downloads"]["cookie_import"] = True
print("cookie: activated and stored in Redis")

View File

@ -4,14 +4,8 @@ Functionality:
- called via user input
"""
from home.src.download.subscriptions import (
ChannelSubscription,
PlaylistSubscription,
)
from home.src.index.playlist import YoutubePlaylist
from home.src.ta.ta_redis import RedisArchivist
from home.src.ta.urlparser import Parser
from home.tasks import run_restore_backup, subscribe_to
from home.tasks import run_restore_backup
class PostData:
@ -36,14 +30,11 @@ class PostData:
exec_map = {
"change_view": self._change_view,
"change_grid": self._change_grid,
"unsubscribe": self._unsubscribe,
"subscribe": self._subscribe,
"sort_order": self._sort_order,
"hide_watched": self._hide_watched,
"show_subed_only": self._show_subed_only,
"show_ignored_only": self._show_ignored_only,
"db-restore": self._db_restore,
"delete-playlist": self._delete_playlist,
}
return exec_map[self.to_exec]
@ -67,34 +58,6 @@ class PostData:
RedisArchivist().set_message(key, {"status": grid_items})
return {"success": True}
def _unsubscribe(self):
"""unsubscribe from channels or playlists"""
id_unsub = self.exec_val
print(f"{id_unsub}: unsubscribe")
to_unsub_list = Parser(id_unsub).parse()
for to_unsub in to_unsub_list:
unsub_type = to_unsub["type"]
unsub_id = to_unsub["url"]
if unsub_type == "playlist":
PlaylistSubscription().change_subscribe(
unsub_id, subscribe_status=False
)
elif unsub_type == "channel":
ChannelSubscription().change_subscribe(
unsub_id, channel_subscribed=False
)
else:
raise ValueError("failed to process " + id_unsub)
return {"success": True}
def _subscribe(self):
"""subscribe to channel or playlist, called from js buttons"""
id_sub = self.exec_val
print(f"{id_sub}: subscribe")
subscribe_to.delay(id_sub)
return {"success": True}
def _sort_order(self):
"""change the sort between published to downloaded"""
sort_order = {"status": self.exec_val}
@ -139,16 +102,3 @@ class PostData:
filename = self.exec_val
run_restore_backup.delay(filename)
return {"success": True}
def _delete_playlist(self):
"""delete playlist, only metadata or incl all videos"""
playlist_dict = self.exec_val
playlist_id = playlist_dict["playlist-id"]
playlist_action = playlist_dict["playlist-action"]
print(f"{playlist_id}: delete playlist {playlist_action}")
if playlist_action == "metadata":
YoutubePlaylist(playlist_id).delete_metadata()
elif playlist_action == "all":
YoutubePlaylist(playlist_id).delete_videos_playlist()
return {"success": True}

View File

@ -100,7 +100,7 @@ class AppConfig:
self.config[config_dict][config_value] = to_write
updated.append((config_value, to_write))
RedisArchivist().set_message("config", self.config)
RedisArchivist().set_message("config", self.config, save=True)
return updated
@staticmethod
@ -112,7 +112,7 @@ class AppConfig:
message = {"status": value}
redis_key = f"{user_id}:{key}"
RedisArchivist().set_message(redis_key, message)
RedisArchivist().set_message(redis_key, message, save=True)
def get_colors(self):
"""overwrite config if user has set custom values"""
@ -225,7 +225,7 @@ class ScheduleBuilder:
to_write = value
redis_config["scheduler"][key] = to_write
RedisArchivist().set_message("config", redis_config)
RedisArchivist().set_message("config", redis_config, save=True)
mess_dict = {
"status": self.MSG,
"level": "info",

View File

@ -15,7 +15,12 @@ import requests
def ignore_filelist(filelist: list[str]) -> list[str]:
"""ignore temp files for os.listdir sanitizer"""
to_ignore = ["Icon\r\r", "Temporary Items", "Network Trash Folder"]
to_ignore = [
"@eaDir",
"Icon\r\r",
"Network Trash Folder",
"Temporary Items",
]
cleaned: list[str] = []
for file_name in filelist:
if file_name.startswith(".") or file_name in to_ignore:
@ -110,7 +115,7 @@ def clear_dl_cache(config: dict) -> int:
"""clear leftover files from dl cache"""
print("clear download cache")
cache_dir = os.path.join(config["application"]["cache_dir"], "download")
leftover_files = os.listdir(cache_dir)
leftover_files = ignore_filelist(os.listdir(cache_dir))
for cached in leftover_files:
to_delete = os.path.join(cache_dir, cached)
os.remove(to_delete)

View File

@ -41,6 +41,7 @@ class RedisArchivist(RedisBase):
message: dict,
path: str = ".",
expire: bool | int = False,
save: bool = False,
) -> None:
"""write new message to redis"""
self.conn.execute_command(
@ -54,6 +55,16 @@ class RedisArchivist(RedisBase):
secs = expire
self.conn.execute_command("EXPIRE", self.NAME_SPACE + key, secs)
if save:
self.bg_save()
def bg_save(self) -> None:
"""save to aof"""
try:
self.conn.bgsave()
except redis.exceptions.ResponseError:
pass
def get_message(self, key: str) -> dict:
"""get message dict from redis"""
reply = self.conn.execute_command("JSON.GET", self.NAME_SPACE + key)

View File

@ -343,9 +343,12 @@ def re_sync_thumbs(self):
@shared_task(bind=True, name="subscribe_to", base=BaseTask)
def subscribe_to(self, url_str):
"""take a list of urls to subscribe to"""
SubscriptionHandler(url_str, task=self).subscribe()
def subscribe_to(self, url_str: str, expected_type: str | bool = False):
"""
take a list of urls to subscribe to
optionally validate expected_type channel / playlist
"""
SubscriptionHandler(url_str, task=self).subscribe(expected_type)
@shared_task(bind=True, name="index_playlists", base=BaseTask)

View File

@ -66,9 +66,9 @@
<div>
<p>Last refreshed: {{ channel.source.channel_last_refresh }}</p>
{% if channel.source.channel_subscribed %}
<button class="unsubscribe" type="button" id="{{ channel.source.channel_id }}" onclick="unsubscribe(this.id)" title="Unsubscribe from {{ channel.source.channel_name }}">Unsubscribe</button>
<button class="unsubscribe" type="button" data-type="channel" data-subscribe="" data-id="{{ channel.source.channel_id }}" onclick="subscribeStatus(this)" title="Unsubscribe from {{ channel.source.channel_name }}">Unsubscribe</button>
{% else %}
<button type="button" id="{{ channel.source.channel_id }}" onclick="subscribe(this.id)" title="Subscribe to {{ channel.source.channel_name }}">Subscribe</button>
<button type="button" data-type="channel" data-subscribe="true" data-id="{{ channel.source.channel_id }}" onclick="subscribeStatus(this)" title="Subscribe to {{ channel.source.channel_name }}">Subscribe</button>
{% endif %}
</div>
</div>

View File

@ -38,9 +38,9 @@
<p>Subscribers: {{ channel_info.channel_subs|intcomma }}</p>
{% endif %}
{% if channel_info.channel_subscribed %}
<button class="unsubscribe" type="button" id="{{ channel_info.channel_id }}" onclick="unsubscribe(this.id)" title="Unsubscribe from {{ channel_info.channel_name }}">Unsubscribe</button>
<button class="unsubscribe" type="button" data-type="channel" data-subscribe="" data-id="{{ channel_info.channel_id }}" onclick="subscribeStatus(this)" title="Unsubscribe from {{ channel_info.channel_name }}">Unsubscribe</button>
{% else %}
<button type="button" id="{{ channel_info.channel_id }}" onclick="subscribe(this.id)" title="Subscribe to {{ channel_info.channel_name }}">Subscribe</button>
<button type="button" data-type="channel" data-subscribe="true" data-id="{{ channel_info.channel_id }}" onclick="subscribeStatus(this)" title="Subscribe to {{ channel_info.channel_name }}">Subscribe</button>
{% endif %}
</div>
</div>

View File

@ -54,9 +54,9 @@
<a href="{% url 'playlist_id' playlist.source.playlist_id %}"><h2>{{ playlist.source.playlist_name }}</h2></a>
<p>Last refreshed: {{ playlist.source.playlist_last_refresh }}</p>
{% if playlist.source.playlist_subscribed %}
<button class="unsubscribe" type="button" id="{{ playlist.source.playlist_id }}" onclick="unsubscribe(this.id)" title="Unsubscribe from {{ playlist.source.playlist_name }}">Unsubscribe</button>
<button class="unsubscribe" type="button" data-type="playlist" data-subscribe="" data-id="{{ playlist.source.playlist_id }}" onclick="subscribeStatus(this)" title="Unsubscribe from {{ playlist.source.playlist_name }}">Unsubscribe</button>
{% else %}
<button type="button" id="{{ playlist.source.playlist_id }}" onclick="subscribe(this.id)" title="Subscribe to {{ playlist.source.playlist_name }}">Subscribe</button>
<button type="button" data-type="playlist" data-subscribe="true" data-id="{{ playlist.source.playlist_id }}" onclick="subscribeStatus(this)" title="Subscribe to {{ playlist.source.playlist_name }}">Subscribe</button>
{% endif %}
</div>
</div>

View File

@ -49,9 +49,9 @@
<a href="{% url 'playlist_id' playlist.source.playlist_id %}"><h2>{{ playlist.source.playlist_name }}</h2></a>
<p>Last refreshed: {{ playlist.source.playlist_last_refresh }}</p>
{% if playlist.source.playlist_subscribed %}
<button class="unsubscribe" type="button" id="{{ playlist.source.playlist_id }}" onclick="unsubscribe(this.id)" title="Unsubscribe from {{ playlist.source.playlist_name }}">Unsubscribe</button>
<button class="unsubscribe" type="button" data-type="playlist" data-subscribe="" data-id="{{ playlist.source.playlist_id }}" onclick="subscribeStatus(this)" title="Unsubscribe from {{ playlist.source.playlist_name }}">Unsubscribe</button>
{% else %}
<button type="button" id="{{ playlist.source.playlist_id }}" onclick="subscribe(this.id)" title="Subscribe to {{ playlist.source.playlist_name }}">Subscribe</button>
<button type="button" data-type="playlist" data-subscribe="true" data-id="{{ playlist.source.playlist_id }}" onclick="subscribeStatus(this)" title="Subscribe to {{ playlist.source.playlist_name }}">Subscribe</button>
{% endif %}
</div>
</div>

View File

@ -27,9 +27,9 @@
<p>Last refreshed: {{ playlist_info.playlist_last_refresh }}</p>
<p>Playlist:
{% if playlist_info.playlist_subscribed %}
<button class="unsubscribe" type="button" id="{{ playlist_info.playlist_id }}" onclick="unsubscribe(this.id)" title="Unsubscribe from {{ playlist_info.playlist_name }}">Unsubscribe</button>
<button class="unsubscribe" type="button" data-type="playlist" data-subscribe="" data-id="{{ playlist_info.playlist_id }}" onclick="subscribeStatus(this)" title="Unsubscribe from {{ playlist_info.playlist_name }}">Unsubscribe</button>
{% else %}
<button type="button" id="{{ playlist_info.playlist_id }}" onclick="subscribe(this.id)" title="Subscribe to {{ playlist_info.playlist_name }}">Subscribe</button>
<button type="button" data-type="playlist" data-subscribe="true" data-id="{{ playlist_info.playlist_id }}" onclick="subscribeStatus(this)" title="Subscribe to {{ playlist_info.playlist_name }}">Subscribe</button>
{% endif %}
</p>
{% if playlist_info.playlist_active %}
@ -40,8 +40,8 @@
<button onclick="deleteConfirm()" id="delete-item">Delete Playlist</button>
<div class="delete-confirm" id="delete-button">
<span>Delete {{ playlist_info.playlist_name }}?</span>
<button onclick="deletePlaylist(this)" data-action="metadata" data-id="{{ playlist_info.playlist_id }}">Delete metadata</button>
<button onclick="deletePlaylist(this)" data-action="all" class="danger-button" data-id="{{ playlist_info.playlist_id }}">Delete all</button><br>
<button onclick="deletePlaylist(this)" data-action="" data-id="{{ playlist_info.playlist_id }}">Delete metadata</button>
<button onclick="deletePlaylist(this)" data-action="delete-videos" class="danger-button" data-id="{{ playlist_info.playlist_id }}">Delete all</button><br>
<button onclick="cancelDelete()">Cancel</button>
</div>
</div>

View File

@ -736,7 +736,7 @@ class ChannelView(ArchivistResultsView):
if subscribe_form.is_valid():
url_str = request.POST.get("subscribe")
print(url_str)
subscribe_to.delay(url_str)
subscribe_to.delay(url_str, expected_type="channel")
sleep(1)
return redirect("channel", permanent=True)
@ -879,7 +879,7 @@ class PlaylistView(ArchivistResultsView):
if subscribe_form.is_valid():
url_str = request.POST.get("subscribe")
print(url_str)
subscribe_to.delay(url_str)
subscribe_to.delay(url_str, expected_type="playlist")
sleep(1)
return redirect("playlist")

View File

@ -1,13 +1,13 @@
apprise==1.4.5
celery==5.3.1
Django==4.2.3
django-auth-ldap==4.4.0
Django==4.2.4
django-auth-ldap==4.5.0
django-cors-headers==4.2.0
djangorestframework==3.14.0
Pillow==10.0.0
redis==4.6.0
redis==5.0.0
requests==2.31.0
ryd-client==0.0.6
uWSGI==2.0.21
uWSGI==2.0.22
whitenoise==6.5.0
yt_dlp==2023.7.6

View File

@ -71,20 +71,27 @@ function isWatchedButton(button) {
}, 1000);
}
function unsubscribe(id_unsub) {
let payload = JSON.stringify({ unsubscribe: id_unsub });
sendPost(payload);
function subscribeStatus(subscribeButton) {
let id = subscribeButton.getAttribute('data-id');
let type = subscribeButton.getAttribute('data-type');
let subscribe = Boolean(subscribeButton.getAttribute('data-subscribe'));
let apiEndpoint;
let data;
if (type === 'channel') {
apiEndpoint = '/api/channel/';
data = { data: [{ channel_id: id, channel_subscribed: subscribe }] };
} else if (type === 'playlist') {
apiEndpoint = '/api/playlist/';
data = { data: [{ playlist_id: id, playlist_subscribed: subscribe }] };
}
apiRequest(apiEndpoint, 'POST', data);
let message = document.createElement('span');
message.innerText = 'You are unsubscribed.';
document.getElementById(id_unsub).replaceWith(message);
}
function subscribe(id_sub) {
let payload = JSON.stringify({ subscribe: id_sub });
sendPost(payload);
let message = document.createElement('span');
message.innerText = 'You are subscribed.';
document.getElementById(id_sub).replaceWith(message);
if (subscribe) {
message.innerText = 'You are subscribed.';
} else {
message.innerText = 'You are unsubscribed.';
}
subscribeButton.replaceWith(message);
}
function changeView(image) {
@ -374,13 +381,11 @@ function deleteChannel(button) {
function deletePlaylist(button) {
let playlist_id = button.getAttribute('data-id');
let playlist_action = button.getAttribute('data-action');
let payload = JSON.stringify({
'delete-playlist': {
'playlist-id': playlist_id,
'playlist-action': playlist_action,
},
});
sendPost(payload);
let apiEndpoint = `/api/playlist/${playlist_id}/`;
if (playlist_action === 'delete-videos') {
apiEndpoint += '?delete-videos=true';
}
apiRequest(apiEndpoint, 'DELETE');
setTimeout(function () {
window.location.replace('/playlist/');
}, 1000);
@ -1057,9 +1062,9 @@ function createChannel(channel, viewStyle) {
const channelLastRefresh = channel.channel_last_refresh;
let button;
if (channel.channel_subscribed) {
button = `<button class="unsubscribe" type="button" id="${channelId}" onclick="unsubscribe(this.id)" title="Unsubscribe from ${channelName}">Unsubscribe</button>`;
button = `<button class="unsubscribe" type="button" data-id="${channelId}" data-subscribe="" data-type="channel" onclick="subscribeStatus(this)" title="Unsubscribe from ${channelName}">Unsubscribe</button>`;
} else {
button = `<button type="button" id="${channelId}" onclick="subscribe(this.id)" title="Subscribe to ${channelName}">Subscribe</button>`;
button = `<button type="button" data-id="${channelId}" data-subscribe="true" data-type="channel" onclick="subscribeStatus(this)" title="Subscribe to ${channelName}">Subscribe</button>`;
}
// build markup
const markup = `
@ -1103,9 +1108,9 @@ function createPlaylist(playlist, viewStyle) {
const playlistLastRefresh = playlist.playlist_last_refresh;
let button;
if (playlist.playlist_subscribed) {
button = `<button class="unsubscribe" type="button" id="${playlistId}" onclick="unsubscribe(this.id)" title="Unsubscribe from ${playlistName}">Unsubscribe</button>`;
button = `<button class="unsubscribe" type="button" data-id="${playlistId}" data-subscribe="" data-type="playlist" onclick="subscribeStatus(this)" title="Unsubscribe from ${playlistName}">Unsubscribe</button>`;
} else {
button = `<button type="button" id="${playlistId}" onclick="subscribe(this.id)" title="Subscribe to ${playlistName}">Subscribe</button>`;
button = `<button type="button" data-id="${playlistId}" data-subscribe="true" data-type="playlist" onclick="subscribeStatus(this)" title="Subscribe to ${playlistName}">Subscribe</button>`;
}
const markup = `
<div class="playlist-thumbnail">