563 lines
19 KiB
Python
563 lines
19 KiB
Python
"""
|
|
Functionality:
|
|
- all views for home app
|
|
- process post data recieved from frontend via ajax
|
|
"""
|
|
|
|
import urllib.parse
|
|
import json
|
|
|
|
from datetime import datetime
|
|
from time import sleep
|
|
|
|
import requests
|
|
|
|
from django.shortcuts import render, redirect
|
|
from django.http import JsonResponse
|
|
from django.views import View
|
|
from django.utils.http import urlencode
|
|
|
|
from home.src.download import PendingList, ChannelSubscription
|
|
from home.src.searching import SearchHandler, Pagination
|
|
from home.src.config import AppConfig
|
|
from home.src.helper import (
|
|
process_url_list,
|
|
get_dl_message,
|
|
get_message,
|
|
set_message
|
|
)
|
|
from home.tasks import (
|
|
update_subscribed,
|
|
download_pending,
|
|
extrac_dl,
|
|
download_single
|
|
)
|
|
|
|
|
|
class HomeView(View):
|
|
""" handle home page and video search post functionality """
|
|
|
|
CONFIG = AppConfig().config
|
|
ES_URL = CONFIG['application']['es_url']
|
|
|
|
def get(self, request):
|
|
""" return home search results """
|
|
colors, sort_order, hide_watched = self.read_config()
|
|
# handle search
|
|
search_get = request.GET.get('search', False)
|
|
if search_get:
|
|
search_encoded = urllib.parse.quote(search_get)
|
|
else:
|
|
search_encoded = False
|
|
# define page size
|
|
page_get = int(request.GET.get('page', 0))
|
|
pagination_handler = Pagination(page_get, search_encoded)
|
|
|
|
url = self.ES_URL + '/ta_video/_search'
|
|
|
|
data = self.build_data(
|
|
pagination_handler, sort_order, search_get, hide_watched
|
|
)
|
|
|
|
search = SearchHandler(url, data)
|
|
videos_hits = search.get_data()
|
|
max_hits = search.max_hits
|
|
pagination_handler.validate(max_hits)
|
|
context = {
|
|
'videos': videos_hits,
|
|
'pagination': pagination_handler.pagination,
|
|
'sortorder': sort_order,
|
|
'hide_watched': hide_watched,
|
|
'colors': colors
|
|
}
|
|
return render(request, 'home/home.html', context)
|
|
|
|
@staticmethod
|
|
def build_data(pagination_handler, sort_order, search_get, hide_watched):
|
|
""" build the data dict for the search query """
|
|
page_size = pagination_handler.pagination['page_size']
|
|
page_from = pagination_handler.pagination['page_from']
|
|
data = {
|
|
"size": page_size, "from": page_from, "query": {"match_all": {}},
|
|
"sort": [
|
|
{"published": {"order": "desc"}},
|
|
{"date_downloaded": {"order": "desc"}}
|
|
]
|
|
}
|
|
# define sort
|
|
if sort_order == 'downloaded':
|
|
del data['sort'][0]
|
|
if search_get:
|
|
del data['sort']
|
|
if hide_watched:
|
|
data['query'] = {"term": {"player.watched": {"value": False}}}
|
|
if search_get:
|
|
query = {
|
|
"multi_match": {
|
|
"query": search_get,
|
|
"fields": ["title", "channel.channel_name", "tags"],
|
|
"type": "cross_fields",
|
|
"operator": "and"
|
|
}
|
|
}
|
|
data['query'] = query
|
|
|
|
return data
|
|
|
|
@staticmethod
|
|
def read_config():
|
|
""" read needed values from redis """
|
|
config_handler = AppConfig().config
|
|
colors = config_handler['application']['colors']
|
|
sort_order = get_message('sort_order')
|
|
hide_watched = get_message('hide_watched')
|
|
return colors, sort_order, hide_watched
|
|
|
|
@staticmethod
|
|
def post(request):
|
|
""" handle post from search form """
|
|
post_data = dict(request.POST)
|
|
search_query = post_data['videoSearch'][0]
|
|
search_url = '/?' + urlencode({'search': search_query})
|
|
return redirect(search_url, permanent=True)
|
|
|
|
|
|
class AboutView(View):
|
|
""" resolves to /about/
|
|
show helpful how to information
|
|
"""
|
|
|
|
@staticmethod
|
|
def get(request):
|
|
""" handle http get """
|
|
config = AppConfig().config
|
|
colors = config['application']['colors']
|
|
context = {
|
|
'title': 'About',
|
|
'colors': colors
|
|
}
|
|
return render(request, 'home/about.html', context)
|
|
|
|
|
|
class DownloadView(View):
|
|
"""
|
|
resolves to /download/
|
|
takes POST for downloading youtube links
|
|
"""
|
|
|
|
@staticmethod
|
|
def get(request):
|
|
""" handle get requests """
|
|
config = AppConfig().config
|
|
colors = config['application']['colors']
|
|
pending_handler = PendingList()
|
|
all_pending, _ = pending_handler.get_all_pending()
|
|
context = {
|
|
'pending': all_pending,
|
|
'title': 'Downloads',
|
|
'colors': colors
|
|
}
|
|
return render(request, 'home/downloads.html', context)
|
|
|
|
@staticmethod
|
|
def post(request):
|
|
""" handle post requests """
|
|
download_post = dict(request.POST)
|
|
if 'vid-url' in download_post.keys():
|
|
url_str = download_post['vid-url']
|
|
print('adding to queue')
|
|
youtube_ids = process_url_list(url_str)
|
|
print(youtube_ids)
|
|
extrac_dl.delay(youtube_ids)
|
|
|
|
sleep(2)
|
|
return redirect('downloads', permanent=True)
|
|
|
|
|
|
class ChannelIdView(View):
|
|
""" display single channel page from channel_id """
|
|
|
|
def get(self, request, channel_id_detail):
|
|
""" get method """
|
|
es_url, colors = self.read_config()
|
|
context = self.get_channel_videos(request, channel_id_detail, es_url)
|
|
context.update({'colors': colors})
|
|
return render(request, 'home/channel_id.html', context)
|
|
|
|
@staticmethod
|
|
def read_config():
|
|
""" read config file """
|
|
config = AppConfig().config
|
|
es_url = config['application']['es_url']
|
|
colors = config['application']['colors']
|
|
return es_url, colors
|
|
|
|
def get_channel_videos(self, request, channel_id_detail, es_url):
|
|
""" get channel from video index """
|
|
page_get = int(request.GET.get('page', 0))
|
|
pagination_handler = Pagination(page_get)
|
|
# get data
|
|
url = es_url + '/ta_video/_search'
|
|
data = self.build_data(pagination_handler, channel_id_detail)
|
|
search = SearchHandler(url, data)
|
|
videos_hits = search.get_data()
|
|
max_hits = search.max_hits
|
|
if max_hits:
|
|
channel_info = videos_hits[0]['source']['channel']
|
|
channel_name = channel_info['channel_name']
|
|
pagination_handler.validate(max_hits)
|
|
pagination = pagination_handler.pagination
|
|
else:
|
|
# get details from channel index when when no hits
|
|
channel_info, channel_name = self.get_channel_info(
|
|
channel_id_detail, es_url
|
|
)
|
|
videos_hits = False
|
|
pagination = False
|
|
|
|
context = {
|
|
'channel_info': channel_info,
|
|
'videos': videos_hits,
|
|
'max_hits': max_hits,
|
|
'pagination': pagination,
|
|
'title': 'Channel: ' + channel_name,
|
|
}
|
|
|
|
return context
|
|
|
|
@staticmethod
|
|
def build_data(pagination_handler, channel_id_detail):
|
|
""" build data dict for search """
|
|
page_size = pagination_handler.pagination['page_size']
|
|
page_from = pagination_handler.pagination['page_from']
|
|
data = {
|
|
"size": page_size, "from": page_from,
|
|
"query": {
|
|
"term": {"channel.channel_id": {"value": channel_id_detail}}
|
|
},
|
|
"sort": [
|
|
{"published": {"order": "desc"}},
|
|
{"date_downloaded": {"order": "desc"}}
|
|
]
|
|
}
|
|
return data
|
|
|
|
@staticmethod
|
|
def get_channel_info(channel_id_detail, es_url):
|
|
""" get channel info from channel index if no videos """
|
|
url = f'{es_url}/ta_channel/_doc/{channel_id_detail}'
|
|
data = False
|
|
search = SearchHandler(url, data)
|
|
channel_data = search.get_data()
|
|
channel_info = channel_data[0]['source']
|
|
channel_name = channel_info['channel_name']
|
|
return channel_info, channel_name
|
|
|
|
|
|
class ChannelView(View):
|
|
""" handle functionality for channel overview page """
|
|
|
|
def get(self, request):
|
|
""" handle http get requests """
|
|
es_url, colors = self.read_config()
|
|
page_get = int(request.GET.get('page', 0))
|
|
pagination_handler = Pagination(page_get)
|
|
page_size = pagination_handler.pagination['page_size']
|
|
page_from = pagination_handler.pagination['page_from']
|
|
# get
|
|
url = es_url + '/ta_channel/_search'
|
|
data = {
|
|
"size": page_size, "from": page_from, "query": {"match_all": {}},
|
|
"sort": [{"channel_name.keyword": {"order": "asc"}}]
|
|
}
|
|
show_subed_only = get_message('show_subed_only')
|
|
if show_subed_only:
|
|
data['query'] = {"term": {"channel_subscribed": {"value": True}}}
|
|
search = SearchHandler(url, data)
|
|
channel_hits = search.get_data()
|
|
max_hits = search.max_hits
|
|
pagination_handler.validate(search.max_hits)
|
|
context = {
|
|
'channels': channel_hits,
|
|
'max_hits': max_hits,
|
|
'pagination': pagination_handler.pagination,
|
|
'show_subed_only': show_subed_only,
|
|
'title': 'Channels',
|
|
'colors': colors
|
|
}
|
|
return render(request, 'home/channel.html', context)
|
|
|
|
@staticmethod
|
|
def read_config():
|
|
""" read config file """
|
|
config = AppConfig().config
|
|
es_url = config['application']['es_url']
|
|
colors = config['application']['colors']
|
|
return es_url, colors
|
|
|
|
@staticmethod
|
|
def post(request):
|
|
""" handle http post requests """
|
|
subscriptions_post = dict(request.POST)
|
|
print(subscriptions_post)
|
|
subscriptions_post = dict(request.POST)
|
|
if 'subscribe' in subscriptions_post.keys():
|
|
youtube_ids = process_url_list(subscriptions_post['subscribe'])
|
|
if youtube_ids[0]['type'] == 'video':
|
|
youtube_id = youtube_ids[0]['url']
|
|
vid_details = PendingList().get_youtube_details(youtube_id)
|
|
channel_id_sub = vid_details['channel_id']
|
|
else:
|
|
channel_id_sub = youtube_ids[0]['url']
|
|
ChannelSubscription().change_subscribe(
|
|
channel_id_sub, channel_subscribed=True
|
|
)
|
|
print('subscribed to: ' + channel_id_sub)
|
|
|
|
sleep(1)
|
|
return redirect('channel', permanent=True)
|
|
|
|
|
|
class VideoView(View):
|
|
""" display details about a single video """
|
|
|
|
def get(self, request, video_id):
|
|
""" get single video """
|
|
es_url, colors = self.read_config()
|
|
url = f'{es_url}/ta_video/_doc/{video_id}'
|
|
data = None
|
|
look_up = SearchHandler(url, data)
|
|
video_hit = look_up.get_data()
|
|
video_data = video_hit[0]['source']
|
|
video_title = video_data['title']
|
|
context = {
|
|
'video': video_data,
|
|
'title': video_title,
|
|
'colors': colors
|
|
}
|
|
return render(request, 'home/video.html', context)
|
|
|
|
@staticmethod
|
|
def read_config():
|
|
""" read config file """
|
|
config = AppConfig().config
|
|
es_url = config['application']['es_url']
|
|
colors = config['application']['colors']
|
|
return es_url, colors
|
|
|
|
|
|
class SettingsView(View):
|
|
""" handle the settings page """
|
|
|
|
@staticmethod
|
|
def get(request):
|
|
""" read and display current settings """
|
|
config = AppConfig().config
|
|
colors = config['application']['colors']
|
|
|
|
context = {
|
|
'title': 'Settings',
|
|
'config': config,
|
|
'colors': colors
|
|
}
|
|
|
|
return render(request, 'home/settings.html', context)
|
|
|
|
@staticmethod
|
|
def post(request):
|
|
""" handle form post to update settings """
|
|
form_post = dict(request.POST)
|
|
del form_post['csrfmiddlewaretoken']
|
|
print(form_post)
|
|
config_handler = AppConfig()
|
|
config_handler.update_config(form_post)
|
|
|
|
return redirect('settings', permanent=True)
|
|
|
|
|
|
def progress(request):
|
|
# pylint: disable=unused-argument
|
|
""" endpoint for download progress ajax calls """
|
|
config = AppConfig().config
|
|
cache_dir = config['application']['cache_dir']
|
|
json_data = get_dl_message(cache_dir)
|
|
return JsonResponse(json_data)
|
|
|
|
|
|
def process(request):
|
|
""" handle all the buttons calls via POST ajax """
|
|
if request.method == 'POST':
|
|
post_dict = json.loads(request.body.decode())
|
|
post_handler = PostData(post_dict)
|
|
if post_handler.to_do:
|
|
task_result = post_handler.run_task()
|
|
return JsonResponse(task_result)
|
|
|
|
return JsonResponse({'success': False})
|
|
|
|
|
|
class PostData:
|
|
""" generic post handler from process route """
|
|
|
|
CONFIG = AppConfig().config
|
|
ES_URL = CONFIG['application']['es_url']
|
|
|
|
VALID_KEYS = [
|
|
"watched", "rescan_pending", "ignore", "dl_pending",
|
|
"unsubscribe", "sort_order", "hide_watched", "show_subed_only",
|
|
"channel-search", "video-search", "dlnow"
|
|
]
|
|
|
|
def __init__(self, post_dict):
|
|
self.post_dict = post_dict
|
|
self.to_do = self.validate()
|
|
|
|
def validate(self):
|
|
""" validate the post_dict """
|
|
to_do = []
|
|
for key, value in self.post_dict.items():
|
|
if key in self.VALID_KEYS:
|
|
task_item = {'task': key, 'status': value}
|
|
print(task_item)
|
|
to_do.append(task_item)
|
|
else:
|
|
print(key + ' not a valid key')
|
|
|
|
return to_do
|
|
|
|
def run_task(self):
|
|
""" run through the tasks to do """
|
|
for item in self.to_do:
|
|
task = item['task']
|
|
if task == 'watched':
|
|
youtube_id = item['status']
|
|
self.parse_watched(youtube_id)
|
|
elif task == 'rescan_pending':
|
|
print('download pending videos')
|
|
update_subscribed.delay()
|
|
elif task == 'ignore':
|
|
print('ignore video')
|
|
handler = PendingList()
|
|
ignore_list = item['status']
|
|
handler.ignore_from_pending([ignore_list])
|
|
elif task == 'dl_pending':
|
|
print('download pending')
|
|
download_pending.delay()
|
|
elif task == 'vid-url':
|
|
status_str = item['status']
|
|
print(urllib.parse.quote(status_str))
|
|
elif task == 'unsubscribe':
|
|
channel_id_unsub = item['status']
|
|
print('unsubscribe from ' + channel_id_unsub)
|
|
ChannelSubscription().change_subscribe(
|
|
channel_id_unsub, channel_subscribed=False
|
|
)
|
|
elif task == 'sort_order':
|
|
sort_order = item['status']
|
|
set_message('sort_order', sort_order, expire=False)
|
|
elif task == 'hide_watched':
|
|
hide_watched = bool(int(item['status']))
|
|
print(item['status'])
|
|
set_message('hide_watched', hide_watched, expire=False)
|
|
elif task == 'show_subed_only':
|
|
show_subed_only = bool(int(item['status']))
|
|
print(show_subed_only)
|
|
set_message('show_subed_only', show_subed_only, expire=False)
|
|
elif task == 'channel-search':
|
|
search_query = item['status']
|
|
print('searching for: ' + search_query)
|
|
search_results = self.search_channels(search_query)
|
|
return search_results
|
|
elif task == 'video-search':
|
|
search_query = item['status']
|
|
print('searching for: ' + search_query)
|
|
search_results = self.search_videos(search_query)
|
|
return search_results
|
|
elif task == 'dlnow':
|
|
youtube_id = item['status']
|
|
print('downloading: ' + youtube_id)
|
|
download_single(youtube_id)
|
|
return {'success': True}
|
|
|
|
def search_channels(self, search_query):
|
|
""" fancy searching channels as you type """
|
|
url = self.ES_URL + '/ta_channel/_search'
|
|
data = {
|
|
"size": 10,
|
|
"query": {
|
|
"multi_match": {
|
|
"query": search_query,
|
|
"type": "bool_prefix",
|
|
"fields": [
|
|
"channel_name.search_as_you_type",
|
|
"channel_name._2gram",
|
|
"channel_name._3gram"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
look_up = SearchHandler(url, data, cache=False)
|
|
search_results = look_up.get_data()
|
|
return {'results': search_results}
|
|
|
|
def search_videos(self, search_query):
|
|
""" fancy searching videos as you type """
|
|
url = self.ES_URL + '/ta_video/_search'
|
|
data = {
|
|
"size": 10,
|
|
"query": {
|
|
"multi_match": {
|
|
"query": search_query,
|
|
"type": "bool_prefix",
|
|
"fields": [
|
|
"title.search_as_you_type",
|
|
"title._2gram",
|
|
"title._3gram"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
look_up = SearchHandler(url, data, cache=False)
|
|
search_results = look_up.get_data()
|
|
return {'results': search_results}
|
|
|
|
def parse_watched(self, youtube_id):
|
|
""" marked as watched based on id type """
|
|
es_url = self.ES_URL
|
|
id_type = process_url_list([youtube_id])[0]['type']
|
|
stamp = int(datetime.now().strftime("%s"))
|
|
if id_type == 'video':
|
|
stamp = int(datetime.now().strftime("%s"))
|
|
url = self.ES_URL + '/ta_video/_update/' + youtube_id
|
|
source = {
|
|
"doc": {"player": {"watched": True, "watched_date": stamp}}
|
|
}
|
|
request = requests.post(url, json=source)
|
|
if not request.ok:
|
|
print(request.text)
|
|
elif id_type == 'channel':
|
|
headers = {'Content-type': 'application/json'}
|
|
data = {
|
|
"description": youtube_id,
|
|
"processors": [
|
|
{"set": {"field": "player.watched", "value": True}},
|
|
{"set": {"field": "player.watched_date", "value": stamp}}
|
|
]
|
|
}
|
|
payload = json.dumps(data)
|
|
url = es_url + '/_ingest/pipeline/' + youtube_id
|
|
request = requests.put(url, data=payload, headers=headers)
|
|
if not request.ok:
|
|
print(request.text)
|
|
# apply pipeline
|
|
must_list = [
|
|
{"term": {"channel.channel_id": {"value": youtube_id}}},
|
|
{"term": {"player.watched": {"value": False}}}
|
|
]
|
|
data = {"query": {"bool": {"must": must_list}}}
|
|
payload = json.dumps(data)
|
|
url = f'{es_url}/ta_video/_update_by_query?pipeline={youtube_id}'
|
|
request = requests.post(url, data=payload, headers=headers)
|
|
if not request.ok:
|
|
print(request.text)
|