tubearchivist/tubearchivist/home/views.py

1067 lines
35 KiB
Python
Raw Normal View History

2021-09-05 17:10:14 +00:00
"""
Functionality:
- all views for home app
- holds base classes to inherit from
2021-09-05 17:10:14 +00:00
"""
import json
2021-09-18 13:02:54 +00:00
import urllib.parse
2021-09-05 17:10:14 +00:00
from time import sleep
2022-07-04 09:44:37 +00:00
from api.src.search_processor import SearchProcess
2022-04-05 11:43:15 +00:00
from django.conf import settings
from django.contrib.auth import login
from django.contrib.auth.forms import AuthenticationForm
2021-09-05 17:10:14 +00:00
from django.http import JsonResponse
2021-09-18 13:02:54 +00:00
from django.shortcuts import redirect, render
from django.views import View
2022-05-24 08:51:58 +00:00
from home.src.download.yt_dlp_base import CookieHandler
from home.src.es.backup import ElasticBackup
2022-07-04 09:44:37 +00:00
from home.src.es.connect import ElasticWrap
from home.src.es.snapshot import ElasticSnapshot
2022-01-22 15:13:37 +00:00
from home.src.frontend.api_calls import PostData
from home.src.frontend.forms import (
2021-10-30 07:14:16 +00:00
AddToQueueForm,
2021-10-29 16:43:19 +00:00
ApplicationSettingsForm,
ChannelOverwriteForm,
2021-10-29 16:43:19 +00:00
CustomAuthForm,
MultiSearchForm,
SchedulerSettingsForm,
SubscribeToChannelForm,
SubscribeToPlaylistForm,
2021-10-29 16:43:19 +00:00
UserSettingsForm,
)
2022-01-22 15:13:37 +00:00
from home.src.frontend.searching import SearchHandler
2022-10-17 06:29:21 +00:00
from home.src.index.channel import YoutubeChannel, channel_overwrites
2022-01-22 15:13:37 +00:00
from home.src.index.generic import Pagination
from home.src.index.playlist import YoutubePlaylist
from home.src.ta.config import AppConfig, ScheduleBuilder
2022-07-21 10:15:36 +00:00
from home.src.ta.helper import UrlListParser, time_parser
2022-01-22 15:13:37 +00:00
from home.src.ta.ta_redis import RedisArchivist
from home.tasks import extrac_dl, index_channel_playlists, subscribe_to
2022-01-11 07:15:36 +00:00
from rest_framework.authtoken.models import Token
2021-09-05 17:10:14 +00:00
class ArchivistViewConfig(View):
"""base view class to generate initial config context"""
def __init__(self, view_origin):
super().__init__()
self.view_origin = view_origin
self.user_id = False
self.user_conf = False
self.default_conf = False
self.context = False
def _get_sort_by(self):
"""return sort_by config var"""
messag_key = f"{self.user_id}:sort_by"
sort_by = self.user_conf.get_message(messag_key)["status"]
if not sort_by:
sort_by = self.default_conf["archive"]["sort_by"]
2021-09-05 17:10:14 +00:00
return sort_by
2021-09-05 17:10:14 +00:00
def _get_sort_order(self):
"""return sort_order config var"""
sort_order_key = f"{self.user_id}:sort_order"
sort_order = self.user_conf.get_message(sort_order_key)["status"]
if not sort_order:
sort_order = self.default_conf["archive"]["sort_order"]
return sort_order
def _get_view_style(self):
"""return view_style config var"""
view_key = f"{self.user_id}:view:{self.view_origin}"
view_style = self.user_conf.get_message(view_key)["status"]
if not view_style:
view_style = self.default_conf["default_view"][self.view_origin]
return view_style
2022-05-28 06:57:29 +00:00
def _get_grid_items(self):
"""return items per row to show in grid view"""
grid_key = f"{self.user_id}:grid_items"
grid_items = self.user_conf.get_message(grid_key)["status"]
if not grid_items:
grid_items = self.default_conf["default_view"]["grid_items"]
return grid_items
def get_all_view_styles(self):
"""get dict of all view stiles for search form"""
all_keys = ["channel", "playlist", "home"]
all_styles = {}
for view_origin in all_keys:
view_key = f"{self.user_id}:view:{view_origin}"
view_style = self.user_conf.get_message(view_key)["status"]
if not view_style:
view_style = self.default_conf["default_view"][view_origin]
all_styles[view_origin] = view_style
return all_styles
def _get_hide_watched(self):
hide_watched_key = f"{self.user_id}:hide_watched"
hide_watched = self.user_conf.get_message(hide_watched_key)["status"]
return hide_watched
def _get_show_ignore_only(self):
ignored_key = f"{self.user_id}:show_ignored_only"
show_ignored_only = self.user_conf.get_message(ignored_key)["status"]
return show_ignored_only
def _get_show_subed_only(self):
sub_only_key = f"{self.user_id}:show_subed_only"
show_subed_only = self.user_conf.get_message(sub_only_key)["status"]
return show_subed_only
def config_builder(self, user_id):
"""build default context for every view"""
self.user_id = user_id
self.user_conf = RedisArchivist()
2022-04-08 08:15:59 +00:00
self.default_conf = AppConfig(self.user_id).config
self.context = {
"colors": self.default_conf["application"]["colors"],
"cast": self.default_conf["application"]["enable_cast"],
"sort_by": self._get_sort_by(),
"sort_order": self._get_sort_order(),
"view_style": self._get_view_style(),
2022-05-28 06:57:29 +00:00
"grid_items": self._get_grid_items(),
"hide_watched": self._get_hide_watched(),
"show_ignored_only": self._get_show_ignore_only(),
"show_subed_only": self._get_show_subed_only(),
2022-04-05 11:43:15 +00:00
"version": settings.TA_VERSION,
}
class ArchivistResultsView(ArchivistViewConfig):
"""View class to inherit from when searching data in es"""
view_origin = False
es_search = False
def __init__(self):
super().__init__(self.view_origin)
self.pagination_handler = False
self.search_get = False
self.data = False
self.sort_by = False
def _sort_by_overwrite(self):
"""overwrite sort by key to match with es keys"""
sort_by_map = {
"views": "stats.view_count",
"likes": "stats.like_count",
"downloaded": "date_downloaded",
"published": "published",
}
sort_by = sort_by_map[self.context["sort_by"]]
return sort_by
@staticmethod
def _url_encode(search_get):
"""url encode search form request"""
2021-09-05 17:10:14 +00:00
if search_get:
search_encoded = urllib.parse.quote(search_get)
else:
search_encoded = False
return search_encoded
def _initial_data(self):
"""add initial data dict"""
sort_order = self.context["sort_order"]
data = {
"size": self.pagination_handler.pagination["page_size"],
"from": self.pagination_handler.pagination["page_from"],
"query": {"match_all": {}},
"sort": [{self.sort_by: {"order": sort_order}}],
}
self.data = data
2022-02-24 11:58:26 +00:00
def match_progress(self):
"""add video progress to result context"""
results = RedisArchivist().list_items(f"{self.user_id}:progress:")
if not results or not self.context["results"]:
2022-02-24 14:25:12 +00:00
return
2022-03-12 10:29:34 +00:00
self.context["continue_vids"] = self.get_in_progress(results)
in_progress = {i["youtube_id"]: i["position"] for i in results}
2022-02-24 11:58:26 +00:00
for hit in self.context["results"]:
video = hit["source"]
2022-03-12 10:29:34 +00:00
if video["youtube_id"] in in_progress:
played_sec = in_progress.get(video["youtube_id"])
2022-02-24 11:58:26 +00:00
total = video["player"]["duration"]
if not total:
total = played_sec * 2
2022-02-24 11:58:26 +00:00
video["player"]["progress"] = 100 * (played_sec / total)
2022-03-12 10:29:34 +00:00
def get_in_progress(self, results):
"""get all videos in progress"""
ids = [{"match": {"youtube_id": i.get("youtube_id")}} for i in results]
data = {
"size": self.default_conf["archive"]["page_size"],
"query": {"bool": {"should": ids}},
2022-03-12 13:29:26 +00:00
"sort": [{"published": {"order": "desc"}}],
2022-03-12 10:29:34 +00:00
}
search = SearchHandler(
"ta_video/_search", self.default_conf, data=data
)
videos = search.get_data()
if not videos:
return False
2022-03-12 10:29:34 +00:00
for video in videos:
youtube_id = video["source"]["youtube_id"]
matched = [i for i in results if i["youtube_id"] == youtube_id]
played_sec = matched[0]["position"]
total = video["source"]["player"]["duration"]
if not total:
total = matched[0].get("position") * 2
2022-03-12 10:29:34 +00:00
video["source"]["player"]["progress"] = 100 * (played_sec / total)
return videos
2022-02-24 11:58:26 +00:00
def single_lookup(self, es_path):
"""retrieve a single item from url"""
search = SearchHandler(es_path, config=self.default_conf)
result = search.get_data()[0]["source"]
return result
def initiate_vars(self, request):
"""search in es for vidoe hits"""
self.user_id = request.user.id
self.config_builder(self.user_id)
self.search_get = request.GET.get("search", False)
2022-10-17 11:40:20 +00:00
self.pagination_handler = Pagination(request)
self.sort_by = self._sort_by_overwrite()
self._initial_data()
2021-09-05 17:10:14 +00:00
def find_results(self):
"""add results and pagination to context"""
search = SearchHandler(
self.es_search, config=self.default_conf, data=self.data
)
self.context["results"] = search.get_data()
self.pagination_handler.validate(search.max_hits)
self.context["max_hits"] = search.max_hits
self.context["pagination"] = self.pagination_handler.pagination
class HomeView(ArchivistResultsView):
"""resolves to /
handle home page and video search post functionality
"""
view_origin = "home"
es_search = "ta_video/_search"
def get(self, request):
"""handle get requests"""
self.initiate_vars(request)
self._update_view_data()
self.find_results()
2022-02-24 11:58:26 +00:00
self.match_progress()
return render(request, "home/home.html", self.context)
def _update_view_data(self):
"""update view specific data dict"""
self.data["sort"].extend(
[
{"channel.channel_name.keyword": {"order": "asc"}},
{"title.keyword": {"order": "asc"}},
]
)
if self.context["hide_watched"]:
self.data["query"] = {"term": {"player.watched": {"value": False}}}
if self.search_get:
del self.data["sort"]
2021-09-05 17:10:14 +00:00
query = {
"multi_match": {
"query": self.search_get,
2021-09-05 17:10:14 +00:00
"fields": ["title", "channel.channel_name", "tags"],
"type": "cross_fields",
2021-09-21 09:25:22 +00:00
"operator": "and",
2021-09-05 17:10:14 +00:00
}
}
self.data["query"] = query
2021-09-05 17:10:14 +00:00
2021-10-18 10:14:59 +00:00
class LoginView(View):
"""resolves to /login/
Greeting and login page
"""
SEC_IN_DAY = 60 * 60 * 24
2021-10-29 16:43:19 +00:00
@staticmethod
def get(request):
2021-10-18 10:14:59 +00:00
"""handle get requests"""
2021-10-24 08:34:00 +00:00
failed = bool(request.GET.get("failed"))
2021-10-29 16:43:19 +00:00
colors = AppConfig(request.user.id).colors
form = CustomAuthForm()
2021-10-24 08:34:00 +00:00
context = {"colors": colors, "form": form, "form_error": failed}
2021-10-18 10:14:59 +00:00
return render(request, "home/login.html", context)
def post(self, request):
"""handle login post request"""
form = AuthenticationForm(data=request.POST)
if form.is_valid():
remember_me = request.POST.get("remember_me") or False
if remember_me == "on":
request.session.set_expiry(self.SEC_IN_DAY * 365)
else:
request.session.set_expiry(self.SEC_IN_DAY * 2)
print(f"expire session in {request.session.get_expiry_age()} secs")
2021-10-22 11:23:06 +00:00
next_url = request.POST.get("next") or "home"
user = form.get_user()
login(request, user)
2021-10-22 05:01:30 +00:00
return redirect(next_url)
2021-10-24 08:34:00 +00:00
return redirect("/login?failed=true")
2021-10-18 10:14:59 +00:00
2021-09-05 17:10:14 +00:00
class AboutView(View):
2021-09-21 09:25:22 +00:00
"""resolves to /about/
2021-09-05 17:10:14 +00:00
show helpful how to information
"""
@staticmethod
def get(request):
2021-09-21 09:25:22 +00:00
"""handle http get"""
2022-04-05 11:43:15 +00:00
context = {
"title": "About",
"colors": AppConfig(request.user.id).colors,
"version": settings.TA_VERSION,
}
2021-09-21 09:25:22 +00:00
return render(request, "home/about.html", context)
2021-09-05 17:10:14 +00:00
class DownloadView(ArchivistResultsView):
2021-09-21 09:25:22 +00:00
"""resolves to /download/
2021-09-05 17:10:14 +00:00
takes POST for downloading youtube links
"""
view_origin = "downloads"
es_search = "ta_download/_search"
def get(self, request):
"""handle get request"""
self.initiate_vars(request)
self._update_view_data(request)
self.find_results()
self.context.update(
{
"title": "Downloads",
"add_form": AddToQueueForm(),
"channel_agg_list": self._get_channel_agg(),
}
)
return render(request, "home/downloads.html", self.context)
2021-09-05 17:10:14 +00:00
def _update_view_data(self, request):
"""update downloads view specific data dict"""
if self.context["show_ignored_only"]:
filter_view = "ignore"
else:
filter_view = "pending"
must_list = [{"term": {"status": {"value": filter_view}}}]
channel_filter = request.GET.get("channel", False)
if channel_filter:
must_list.append(
{"term": {"channel_id": {"value": channel_filter}}}
)
2022-10-17 06:29:21 +00:00
channel = YoutubeChannel(channel_filter)
channel.get_from_es()
self.context.update(
{
"channel_filter_id": channel_filter,
"channel_filter_name": channel.json_data["channel_name"],
}
)
self.data.update(
{
"query": {"bool": {"must": must_list}},
"sort": [{"timestamp": {"order": "asc"}}],
}
)
def _get_channel_agg(self):
"""get pending channel with count"""
data = {
"size": 0,
"query": {"term": {"status": {"value": "pending"}}},
"aggs": {
"channel_downloads": {
"multi_terms": {
"size": 30,
"terms": [
{"field": "channel_name.keyword"},
{"field": "channel_id"},
],
"order": {"_count": "desc"},
}
}
},
}
response, _ = ElasticWrap(self.es_search).get(data=data)
buckets = response["aggregations"]["channel_downloads"]["buckets"]
buckets_sorted = []
for i in buckets:
bucket = {
"name": i["key"][0],
"id": i["key"][1],
"count": i["doc_count"],
}
buckets_sorted.append(bucket)
return buckets_sorted
2021-09-05 17:10:14 +00:00
@staticmethod
def post(request):
2021-09-21 09:25:22 +00:00
"""handle post requests"""
2021-10-30 07:14:16 +00:00
to_queue = AddToQueueForm(data=request.POST)
if to_queue.is_valid():
url_str = request.POST.get("vid_url")
print(url_str)
try:
youtube_ids = UrlListParser(url_str).process_list()
except ValueError:
# failed to process
key = "message:add"
print(f"failed to parse: {url_str}")
mess_dict = {
"status": key,
"level": "error",
2021-09-21 09:25:22 +00:00
"title": "Failed to extract links.",
"message": "Not a video, channel or playlist ID or URL",
}
RedisArchivist().set_message(key, mess_dict, expire=True)
2021-09-21 09:25:22 +00:00
return redirect("downloads")
2021-09-05 17:10:14 +00:00
print(youtube_ids)
extrac_dl.delay(youtube_ids)
sleep(2)
2021-09-21 09:25:22 +00:00
return redirect("downloads", permanent=True)
2021-09-05 17:10:14 +00:00
2022-10-17 06:29:21 +00:00
class ChannelIdBaseView(ArchivistResultsView):
"""base class for all channel-id views"""
def get_channel_meta(self, channel_id):
"""get metadata for channel"""
path = f"ta_channel/_doc/{channel_id}"
response, _ = ElasticWrap(path).get()
channel_info = SearchProcess(response).process()
return channel_info
def channel_has_pending(self, channel_id):
"""check if channel has pending videos in queue"""
path = "ta_download/_search"
data = {
"size": 1,
"query": {
"bool": {
"must": [
{"term": {"status": {"value": "pending"}}},
{"term": {"channel_id": {"value": channel_id}}},
]
}
},
}
response, _ = ElasticWrap(path).get(data=data)
self.context.update({"has_pending": bool(response["hits"]["hits"])})
class ChannelIdView(ChannelIdBaseView):
2021-09-21 09:25:22 +00:00
"""resolves to /channel/<channel-id>/
2021-09-09 09:45:46 +00:00
display single channel page from channel_id
"""
2021-09-05 17:10:14 +00:00
view_origin = "home"
es_search = "ta_video/_search"
def get(self, request, channel_id):
"""get request"""
self.initiate_vars(request)
self._update_view_data(channel_id)
self.find_results()
self.match_progress()
2022-10-17 06:29:21 +00:00
self.channel_has_pending(channel_id)
if self.context["results"]:
channel_info = self.context["results"][0]["source"]["channel"]
2021-09-21 09:25:22 +00:00
channel_name = channel_info["channel_name"]
2021-09-05 17:10:14 +00:00
else:
# fall back channel lookup if no videos found
es_path = f"ta_channel/_doc/{channel_id}"
2022-01-08 16:57:51 +00:00
channel_info = self.single_lookup(es_path)
channel_name = channel_info["channel_name"]
2021-09-05 17:10:14 +00:00
self.context.update(
{
"title": "Channel: " + channel_name,
"channel_info": channel_info,
}
)
2021-09-05 17:10:14 +00:00
return render(request, "home/channel_id.html", self.context)
def _update_view_data(self, channel_id):
"""update view specific data dict"""
self.data["query"] = {
"bool": {
"must": [
{"term": {"channel.channel_id": {"value": channel_id}}}
]
}
2021-09-05 17:10:14 +00:00
}
self.data["sort"].append({"title.keyword": {"order": "asc"}})
if self.context["hide_watched"]:
to_append = {"term": {"player.watched": {"value": False}}}
self.data["query"]["bool"]["must"].append(to_append)
2021-09-05 17:10:14 +00:00
@staticmethod
def post(request, channel_id):
"""handle post request"""
print(f"handle post from {channel_id}")
channel_overwrite_form = ChannelOverwriteForm(request.POST)
if channel_overwrite_form.is_valid():
overwrites = channel_overwrite_form.cleaned_data
print(f"{channel_id}: set overwrites {overwrites}")
channel_overwrites(channel_id, overwrites=overwrites)
if overwrites.get("index_playlists") == "1":
index_channel_playlists.delay(channel_id)
sleep(1)
return redirect("channel_id", channel_id, permanent=True)
2021-09-05 17:10:14 +00:00
2022-10-17 06:29:21 +00:00
class ChannelIdAboutView(ChannelIdBaseView):
2022-07-04 09:44:37 +00:00
"""resolves to /channel/<channel-id>/about/
show metadata, handle per channel conf
"""
view_origin = "channel"
def get(self, request, channel_id):
"""handle get request"""
self.initiate_vars(request)
2022-10-17 06:29:21 +00:00
self.channel_has_pending(channel_id)
2022-07-04 09:44:37 +00:00
path = f"ta_channel/_doc/{channel_id}"
response, _ = ElasticWrap(path).get()
channel_info = SearchProcess(response).process()
channel_name = channel_info["channel_name"]
self.context.update(
{
"title": "Channel: About " + channel_name,
"channel_info": channel_info,
"channel_overwrite_form": ChannelOverwriteForm,
}
)
return render(request, "home/channel_id_about.html", self.context)
@staticmethod
def post(request, channel_id):
"""handle post request"""
print(f"handle post from {channel_id}")
channel_overwrite_form = ChannelOverwriteForm(request.POST)
if channel_overwrite_form.is_valid():
overwrites = channel_overwrite_form.cleaned_data
print(f"{channel_id}: set overwrites {overwrites}")
channel_overwrites(channel_id, overwrites=overwrites)
if overwrites.get("index_playlists") == "1":
index_channel_playlists.delay(channel_id)
sleep(1)
return redirect("channel_id_about", channel_id, permanent=True)
2022-10-17 06:29:21 +00:00
class ChannelIdPlaylistView(ChannelIdBaseView):
"""resolves to /channel/<channel-id>/playlist/
show all playlists of channel
"""
view_origin = "playlist"
es_search = "ta_playlist/_search"
def get(self, request, channel_id):
"""handle get request"""
self.initiate_vars(request)
self._update_view_data(channel_id)
self.find_results()
2022-10-17 06:29:21 +00:00
self.channel_has_pending(channel_id)
2022-10-17 06:29:21 +00:00
channel_info = self.get_channel_meta(channel_id)
channel_name = channel_info["channel_name"]
self.context.update(
{
"title": "Channel: Playlists " + channel_name,
"channel_info": channel_info,
}
)
return render(request, "home/channel_id_playlist.html", self.context)
def _update_view_data(self, channel_id):
"""update view specific data dict"""
self.data["sort"] = [{"playlist_name.keyword": {"order": "asc"}}]
2022-07-04 14:12:54 +00:00
must_list = [{"match": {"playlist_channel_id": channel_id}}]
if self.context["show_subed_only"]:
must_list.append({"match": {"playlist_subscribed": True}})
self.data["query"] = {"bool": {"must": must_list}}
2022-07-04 09:44:37 +00:00
class ChannelView(ArchivistResultsView):
2021-09-21 09:25:22 +00:00
"""resolves to /channel/
2021-09-09 09:45:46 +00:00
handle functionality for channel overview page, subscribe to channel,
search as you type for channel name
"""
2021-09-05 17:10:14 +00:00
view_origin = "channel"
es_search = "ta_channel/_search"
2021-09-05 17:10:14 +00:00
def get(self, request):
"""handle get request"""
self.initiate_vars(request)
self._update_view_data()
self.find_results()
self.context.update(
{
"title": "Channels",
"subscribe_form": SubscribeToChannelForm(),
}
)
return render(request, "home/channel.html", self.context)
def _update_view_data(self):
"""update view data dict"""
self.data["sort"] = [{"channel_name.keyword": {"order": "asc"}}]
if self.context["show_subed_only"]:
self.data["query"] = {
"term": {"channel_subscribed": {"value": True}}
}
2021-09-05 17:10:14 +00:00
@staticmethod
def post(request):
2021-09-21 09:25:22 +00:00
"""handle http post requests"""
subscribe_form = SubscribeToChannelForm(data=request.POST)
if subscribe_form.is_valid():
key = "message:subchannel"
message = {
"status": key,
"level": "info",
"title": "Subscribing to Channels",
"message": "Parsing form data",
}
RedisArchivist().set_message(key, message=message, expire=True)
url_str = request.POST.get("subscribe")
print(url_str)
subscribe_to.delay(url_str)
sleep(1)
2021-09-21 09:25:22 +00:00
return redirect("channel", permanent=True)
2021-09-05 17:10:14 +00:00
class PlaylistIdView(ArchivistResultsView):
"""resolves to /playlist/<playlist_id>
show all videos in a playlist
"""
view_origin = "home"
es_search = "ta_video/_search"
def get(self, request, playlist_id):
"""handle get request"""
self.initiate_vars(request)
playlist_info, channel_info = self._get_info(playlist_id)
playlist_name = playlist_info["playlist_name"]
self._update_view_data(playlist_id, playlist_info)
self.find_results()
self.match_progress()
self.context.update(
{
"title": "Playlist: " + playlist_name,
"playlist_info": playlist_info,
"