diff --git a/README.md b/README.md index d3ee806..16a6112 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,9 @@ chown 1000:0 /path/to/mount/point This will match the permissions with the **UID** and **GID** of elasticsearch within the container and should fix the issue. ### Disk usage -The Elasticsearch index will turn to *read only* if the disk usage of the container goes above 95% until the usage drops below 90% again. Similar to that, TubeArchivist will become all sorts of messed up when running out of disk space. There are some error messages in the logs when that happens, but it's best to make sure to have enough disk space before starting to download. +The Elasticsearch index will turn to *read only* if the disk usage of the container goes above 95% until the usage drops below 90% again, you will see error messages like `disk usage exceeded flood-stage watermark`, [link](https://github.com/tubearchivist/tubearchivist#disk-usage). + +Similar to that, TubeArchivist will become all sorts of messed up when running out of disk space. There are some error messages in the logs when that happens, but it's best to make sure to have enough disk space before starting to download. ## Getting Started 1. Go through the **settings** page and look at the available options. Particularly set *Download Format* to your desired video quality before downloading. **Tube Archivist** downloads the best available quality by default. To support iOS or MacOS and some other browsers a compatible format must be specified. For example: diff --git a/tubearchivist/api/README.md b/tubearchivist/api/README.md index c635b11..e2bed71 100644 --- a/tubearchivist/api/README.md +++ b/tubearchivist/api/README.md @@ -20,6 +20,19 @@ headers = {"Authorization": "Token xxxxxxxxxx"} response = requests.get(url, headers=headers) ``` +## Pagination +The list views return a paginate object with the following keys: +- page_size: int current page size set in config +- page_from: int first result idx +- prev_pages: array of ints of previous pages, if available +- current_page: int current page from query +- max_hits: reached: bool if max of 10k results is reached +- last_page: int of last page link +- next_pages: array of ints of next pages +- total_hits: int total results + +Pass page number as a query parameter: `page=2`. Defaults to *0*, `page=1` is redundant and falls back to *0*. If a page query doesn't return any results, you'll get `HTTP 404 Not Found`. + ## Login View Return token and user ID for username and password: POST /api/login diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index a30484e..a92cf60 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -3,6 +3,7 @@ from api.src.search_processor import SearchProcess from home.src.download.queue import PendingInteract from home.src.es.connect import ElasticWrap +from home.src.index.generic import Pagination from home.src.index.video import SponsorBlock from home.src.ta.config import AppConfig from home.src.ta.helper import UrlListParser @@ -25,12 +26,15 @@ class ApiBaseView(APIView): authentication_classes = [SessionAuthentication, TokenAuthentication] permission_classes = [IsAuthenticated] search_base = False + data = False def __init__(self): super().__init__() self.response = {"data": False, "config": AppConfig().config} + self.data = {"query": {"match_all": {}}} self.status_code = False self.context = False + self.pagination_handler = False def get_document(self, document_id): """get single document from es""" @@ -44,20 +48,33 @@ class ApiBaseView(APIView): self.response["data"] = False self.status_code = status_code - def get_paginate(self): - """add pagination detail to response""" - self.response["paginate"] = False + def initiate_pagination(self, request): + """set initial pagination values""" + user_id = request.user.id + page_get = int(request.GET.get("page", 0)) + self.pagination_handler = Pagination(page_get, user_id) + self.data.update( + { + "size": self.pagination_handler.pagination["page_size"], + "from": self.pagination_handler.pagination["page_from"], + } + ) - def get_document_list(self, data): + def get_document_list(self, request): """get a list of results""" print(self.search_base) - response, status_code = ElasticWrap(self.search_base).get(data=data) + self.initiate_pagination(request) + es_handler = ElasticWrap(self.search_base) + response, status_code = es_handler.get(data=self.data) self.response["data"] = SearchProcess(response).process() if self.response["data"]: self.status_code = status_code else: self.status_code = 404 + self.pagination_handler.validate(response["hits"]["total"]["value"]) + self.response["paginate"] = self.pagination_handler.pagination + class VideoApiView(ApiBaseView): """resolves to /api/video// @@ -81,11 +98,9 @@ class VideoApiListView(ApiBaseView): search_base = "ta_video/_search/" def get(self, request): - # pylint: disable=unused-argument """get request""" - data = {"query": {"match_all": {}}} - self.get_document_list(data) - self.get_paginate() + self.data.update({"sort": [{"published": {"order": "desc"}}]}) + self.get_document_list(request) return Response(self.response) @@ -200,11 +215,11 @@ class ChannelApiListView(ApiBaseView): search_base = "ta_channel/_search/" def get(self, request): - # pylint: disable=unused-argument """get request""" - data = {"query": {"match_all": {}}} - self.get_document_list(data) - self.get_paginate() + self.get_document_list(request) + self.data.update( + {"sort": [{"channel_name.keyword": {"order": "asc"}}]} + ) return Response(self.response) @@ -234,13 +249,16 @@ class ChannelApiVideoView(ApiBaseView): search_base = "ta_video/_search/" def get(self, request, channel_id): - # pylint: disable=unused-argument """handle get request""" - data = { - "query": {"term": {"channel.channel_id": {"value": channel_id}}} - } - self.get_document_list(data) - self.get_paginate() + self.data.update( + { + "query": { + "term": {"channel.channel_id": {"value": channel_id}} + }, + "sort": [{"published": {"order": "desc"}}], + } + ) + self.get_document_list(request) return Response(self.response, status=self.status_code) @@ -253,11 +271,11 @@ class PlaylistApiListView(ApiBaseView): search_base = "ta_playlist/_search/" def get(self, request): - # pylint: disable=unused-argument """handle get request""" - data = {"query": {"match_all": {}}} - self.get_document_list(data) - self.get_paginate() + self.data.update( + {"sort": [{"playlist_name.keyword": {"order": "asc"}}]} + ) + self.get_document_list(request) return Response(self.response) @@ -283,13 +301,13 @@ class PlaylistApiVideoView(ApiBaseView): search_base = "ta_video/_search/" def get(self, request, playlist_id): - # pylint: disable=unused-argument """handle get request""" - data = { - "query": {"term": {"playlist.keyword": {"value": playlist_id}}} + self.data["query"] = { + "term": {"playlist.keyword": {"value": playlist_id}} } - self.get_document_list(data) - self.get_paginate() + self.data.update({"sort": [{"published": {"order": "desc"}}]}) + + self.get_document_list(request) return Response(self.response, status=self.status_code) @@ -344,23 +362,18 @@ class DownloadApiListView(ApiBaseView): valid_filter = ["pending", "ignore"] def get(self, request): - # pylint: disable=unused-argument """get request""" query_filter = request.GET.get("filter", False) - data = { - "query": {"match_all": {}}, - "sort": [{"timestamp": {"order": "asc"}}], - } + self.data.update({"sort": [{"timestamp": {"order": "asc"}}]}) if query_filter: if query_filter not in self.valid_filter: message = f"invalid url query filder: {query_filter}" print(message) return Response({"message": message}, status=400) - data["query"] = {"term": {"status": {"value": query_filter}}} + self.data["query"] = {"term": {"status": {"value": query_filter}}} - self.get_document_list(data) - self.get_paginate() + self.get_document_list(request) return Response(self.response) @staticmethod diff --git a/tubearchivist/home/src/index/generic.py b/tubearchivist/home/src/index/generic.py index dcff82b..709dde9 100644 --- a/tubearchivist/home/src/index/generic.py +++ b/tubearchivist/home/src/index/generic.py @@ -147,3 +147,4 @@ class Pagination: ] self.pagination["next_pages"] = next_pages + self.pagination["total_hits"] = total_hits diff --git a/tubearchivist/home/src/index/video.py b/tubearchivist/home/src/index/video.py index 969d716..290e0ce 100644 --- a/tubearchivist/home/src/index/video.py +++ b/tubearchivist/home/src/index/video.py @@ -195,6 +195,11 @@ class SubtitleParser: if flatten: # fix overlapping retiming issue + if "dDurationMs" not in flatten[-1]: + # some events won't have a duration + print(f"failed to parse event without duration: {event}") + continue + last_end = flatten[-1]["tStartMs"] + flatten[-1]["dDurationMs"] if event["tStartMs"] < last_end: joined = flatten[-1]["segs"][0]["utf8"] + "\n" + text diff --git a/tubearchivist/www/src/lib/getDownloads.ts b/tubearchivist/www/src/lib/getDownloads.ts old mode 100644 new mode 100755 index e610a90..02f9ead --- a/tubearchivist/www/src/lib/getDownloads.ts +++ b/tubearchivist/www/src/lib/getDownloads.ts @@ -1,9 +1,10 @@ import { Download } from "../types/download"; -import { DownloadResponse } from "../types/download"; -import { TA_BASE_URL } from "./constants"; +import { getTAUrl } from "./constants"; -export const getDownloads = async (token: string): Promise => { - const response = await fetch(`${TA_BASE_URL}/api/download/`, { +const TA_BASE_URL = getTAUrl(); + +export const getDownloads = async (token: string, filter: boolean, pageNumber: number): Promise => { + const response = await fetch(`${TA_BASE_URL.server}/api/download/?filter=${filter ? 'ignore' : 'pending'}&page=${pageNumber}`, { headers: { Accept: "application/json", "Content-Type": "application/json", @@ -11,20 +12,87 @@ export const getDownloads = async (token: string): Promise => { mode: "no-cors", }, }); - if (!response.ok) { - throw new Error("Error getting download queue information"); + if (response.ok) { + return response.json(); + } else { + // var error: Download = { + // data: null, + // config: null, + // paginate: null, + // message: response.statusText + " (" + response.status + ")", + // } + // error = response.statusText + " (" + response.status + "); + throw new Error(response.statusText + " (" + response.status + ")"); + // return error; } - return response.json(); }; -export const sendDownloads = async (token: string, input: string): Promise => { +export const sendDownloads = async (token: string, input: string): Promise => { var data = { "data": [{ "youtube_id": input, "status": "pending" }] }; - const response = await fetch(`${TA_BASE_URL}/api/download/`, { + const response = await fetch(`${TA_BASE_URL.server}/api/download/`, { + body: JSON.stringify(data), + headers: { + Accept: "application/json", + "Content-Type": "application/json", + Authorization: `Token ${token}`, + mode: "no-cors", + }, + method: "POST" + }); + if (response.ok) { + return response.json(); + } else if (response.status == 400) { + throw new Error("Failed to extract links. Please input IDs or URLs for videos, channels, or playlists."); + } else { + throw new Error("Failed to connect to the API."); + } + +}; + +export const sendDeleteAllQueuedIgnored = async (token: string, filter: string): Promise => { + const response = await fetch(`${TA_BASE_URL.server}/api/download/?filter=${filter}`, { + headers: { + Accept: "application/json", + "Content-Type": "application/json", + Authorization: `Token ${token}`, + mode: "no-cors", + }, + method: "DELETE" + }); + if (!response.ok) { + throw new Error("Error removing all videos."); + // return response.json(); + } + return response.json(); +}; + +export const sendDeleteVideoQueuedIgnored = async (token: string, videoId: string): Promise => { + const response = await fetch(`${TA_BASE_URL.server}/api/download/${videoId}/`, { + headers: { + Accept: "application/json", + "Content-Type": "application/json", + Authorization: `Token ${token}`, + mode: "no-cors", + }, + method: "DELETE" + }); + if (!response.ok) { + throw new Error("Error removing video."); + // return response.json(); + } + return response.json(); +}; + +export const sendMoveVideoQueuedIgnored = async (token: string, videoId: string, status: string): Promise => { + var data = { + "status": status + }; + const response = await fetch(`${TA_BASE_URL.server}/api/download/${videoId}/`, { body: JSON.stringify(data), headers: { Accept: "application/json", @@ -35,7 +103,7 @@ export const sendDownloads = async (token: string, input: string): Promise { {/* {% if channel.source.channel_subs >= 1000000 %} */} -

Subscribers: {channel?.channel_subs}

+

Subscribers: {formatNumbers(channel?.channel_subs)}

{/* {% else %} */} @@ -184,20 +185,20 @@ const Channel: NextPage = () => { className="unsubscribe" type="button" id="{{ channel.source.channel_id }}" - onClick={() => console.log("unsubscribe(this.id)")} - title="Unsubscribe from {{ channel.source.channel_name }}" + onClick={() => console.log("unsubscribe(this.id) -> toggleSubscribe()")} + title={`${channel?.channel_subscribed ? "Unsubscribe from" : "Subscribe to"} ${channel?.channel_name}`} > - Unsubscribe + {channel?.channel_subscribed ? "Unsubscribe" : "Subscribe"} {/* {% else %} */} - + */} {/* {% endif %} */} diff --git a/tubearchivist/www/src/pages/download.tsx b/tubearchivist/www/src/pages/download.tsx index 224b38c..3682de5 100755 --- a/tubearchivist/www/src/pages/download.tsx +++ b/tubearchivist/www/src/pages/download.tsx @@ -5,20 +5,24 @@ import { dehydrate, QueryClient, useQuery } from "react-query"; import { CustomHead } from "../components/CustomHead"; import { Layout } from "../components/Layout"; import NextImage from "next/image"; -import { TA_BASE_URL } from "../lib/constants"; -import { getDownloads } from "../lib/getDownloads"; +import { getDownloads, sendDeleteAllQueuedIgnored, sendDeleteVideoQueuedIgnored, sendMoveVideoQueuedIgnored } from "../lib/getDownloads"; import { sendDownloads } from "../lib/getDownloads"; import RescanIcon from "../images/icon-rescan.svg"; import DownloadIcon from "../images/icon-download.svg"; import AddIcon from "../images/icon-add.svg"; import GridViewIcon from "../images/icon-gridview.svg"; import ListViewIcon from "../images/icon-listview.svg"; +import StopIcon from "../images/icon-stop.svg"; +import CloseIcon from "../images/icon-close.svg"; +import { getTAUrl } from "../lib/constants"; +const TA_BASE_URL = getTAUrl(); type ViewStyle = "grid" | "list"; type IgnoredStatus = boolean; type FormHidden = boolean; - +type ErrorMessage = string; +type PageNumber = number; export const getServerSideProps: GetServerSideProps = async (context) => { const queryClient = new QueryClient(); @@ -33,8 +37,8 @@ export const getServerSideProps: GetServerSideProps = async (context) => { }; } - await queryClient.prefetchQuery(["downloads", session.ta_token.token], () => - getDownloads(session.ta_token.token) + await queryClient.prefetchQuery(["downloads", session.ta_token.token, false, 1], () => + getDownloads(session.ta_token.token, false, 1) ); return { @@ -47,40 +51,77 @@ export const getServerSideProps: GetServerSideProps = async (context) => { const Download: NextPage = () => { const { data: session } = useSession(); + + const [ignoredStatus, setIgnoredStatus] = useState(false); + const [pageNumber, setPageNumber] = useState(1); + const { data: downloads, error, isLoading, + refetch, } = useQuery( - ["downloads", session.ta_token.token], - () => getDownloads(session.ta_token.token), + ["downloads", session.ta_token.token, ignoredStatus, pageNumber], + () => getDownloads(session.ta_token.token, ignoredStatus, pageNumber), { enabled: !!session?.ta_token?.token, + refetchInterval: 1500, + refetchIntervalInBackground: false, } ); - var count = 0; - - const [viewStyle, setViewStyle] = useState(downloads?.config?.default_view?.downloads); - const [ignoredStatus, setIgnoredStatus] = useState(false); const [formHidden, setFormHidden] = useState(true); + const [viewStyle, setViewStyle] = useState(downloads?.config?.default_view?.downloads); + const [errorMessage, setErrorMessage] = useState(null); const handleSetViewstyle = (selectedViewStyle: ViewStyle) => { setViewStyle(selectedViewStyle); }; - + const handleSetIgnoredStatus = (selectedIgnoredStatus: IgnoredStatus) => { setIgnoredStatus(selectedIgnoredStatus); + refetch(); + handleSetPageNumber(1); }; const handleSetFormHidden = (selectedFormHidden: FormHidden) => { setFormHidden(selectedFormHidden); }; + const handleSetErrorMessage = (selectedErrorMessage: ErrorMessage) => { + setErrorMessage(selectedErrorMessage); + }; + + const handleSetPageNumber = (selectedPageNumber: PageNumber) => { + setPageNumber(selectedPageNumber); + }; + const addToDownloadQueue = event => { event.preventDefault(); - sendDownloads(session.ta_token.token, event.target.vid_url.value); - handleSetFormHidden(true); + sendDownloads(session.ta_token.token, event.target.vid_url.value).then(() => { + handleSetErrorMessage(null); + handleSetFormHidden(true); + }) + .catch(error => handleSetErrorMessage(error.message)); + } + const handleMoveVideoQueuedIgnored = (session: string, youtube_id: string, status: string) => { + sendMoveVideoQueuedIgnored(session, youtube_id, status).then(() => { + handleSetErrorMessage(null); + }) + .catch(error => handleSetErrorMessage(error.message)); + } + const handleDeleteVideoQueuedIgnored = (session: string, youtube_id: string) => { + sendDeleteVideoQueuedIgnored(session, youtube_id).then(() => { + handleSetErrorMessage(null); + }) + .catch(error => handleSetErrorMessage(error.message)); + } + + const handleDeleteAllQueuedIgnored = (session: string, filter: string) => { + sendDeleteAllQueuedIgnored(session, filter).then(() => { + handleSetErrorMessage(null); + }) + .catch(error => handleSetErrorMessage(error.message)); } return ( @@ -92,8 +133,66 @@ const Download: NextPage = () => {

Downloads

-
-
+
+ {(error || !downloads?.data) && !isLoading && +
+

API Connection Error

+

+
+ } + {errorMessage && +
+

Failed to add downloads to the queue.

+

{errorMessage}

+
+ } + { + //
+ //

Adding new videos to download queue.

+ //

Extracting lists

+ //

Progress: 0/0

+ //
+ } + { + //
+ //

Rescanning channels and playlists.

+ //

Looking for new videos.

+ //
+ } + { + //
+ //

Downloading: `VIDEO_TITLE`

+ //

processing

+ //

`DOWNLOADED_PERCENTAGE`% of `VIDEO_SIVE``VIDEO_SIZE_UNIT` at `DOWNLOAD_SPEED``DOWNLOAD_SPEED_UNIT` - time left: `DOWNLOAD_TIME_LEFT`

+ //

processing

+ //

Moving

+ //

Completed

+ //
+ } +
+
+ {/* Appears when video is downloading */} + {/*
+ console.log("stopQueue()")} + /> + console.log("killQueue()")} + /> +
*/} +
{ src={RescanIcon} alt="rescan-icon" title="Rescan subscriptions" + // className="rotate-img" // Set when rescanning onClick={() => console.log("rescanPending()")} /> {/* rescan-icon */} @@ -114,6 +214,7 @@ const Download: NextPage = () => { src={DownloadIcon} alt="download-icon" title="Start download" + // className="bounce-img" // Set when video is downloading onClick={() => console.log("dlPending()")} /> {/* download-icon */} @@ -142,80 +243,63 @@ const Download: NextPage = () => {
Show only ignored videos: - {ignoredStatus && -
- handleSetIgnoredStatus(false)} - type="checkbox" - checked - /> - -
- } - {!ignoredStatus && -
- handleSetIgnoredStatus(true)} - type="checkbox" - /> - -
- } +
+ handleSetIgnoredStatus(!ignoredStatus)} + type="checkbox" + checked={ignoredStatus} + /> + +
- handleSetViewstyle("grid")} - /> +
+ handleSetViewstyle("grid")} + /> +
{/* grid view */} - handleSetViewstyle("list")} - /> +
+ handleSetViewstyle("list")} + /> +
{/* list view */}
{ignoredStatus &&

Ignored from download

- +
} {!ignoredStatus &&

Download queue

- +
} - {downloads?.data?.forEach((video) => { - if ((video?.status == "ignore" && ignoredStatus) || (video?.status == "pending" && !ignoredStatus)) { - count++; - } - })} -

Total videos: {count} {!count &&

No videos queued for download. Press rescan subscriptions to check if there are any new videos.

}

-
- {downloads.data && +

Total videos: {downloads?.paginate?.total_hits} {!downloads && !downloads?.message && !ignoredStatus &&

No videos queued for download. Press rescan subscriptions to check if there are any new videos.

}

+
+ {!isLoading && !error && !downloads?.message && downloads?.data?.map((video) => { - count++; - if ((video?.status == "ignore" && ignoredStatus) || (video?.status == "pending" && !ignoredStatus)) { return (
- video_thumb + video_thumb {ignoredStatus && ignored} {/* {% if show_ignored_only %} */} {/* ignored */} @@ -237,8 +321,8 @@ const Download: NextPage = () => { {/*

Published: {{ video.source.published }} | Duration: {{ video.source.duration }} | {{ video.source.youtube_id }}

*/} {ignoredStatus &&
- - + +
} {/* {% if show_ignored_only %} */} @@ -246,8 +330,9 @@ const Download: NextPage = () => { {/* */} {!ignoredStatus &&
- - + + +
} {/* {% else %} */} @@ -257,7 +342,6 @@ const Download: NextPage = () => {
); - } }) } {/* {% if results %} */} @@ -290,7 +374,29 @@ const Download: NextPage = () => { {/*
*/} {/* {% endfor %} */} {/* {% endif %} */} - +
+
+
+
+ {pageNumber != 1 ? handleSetPageNumber(1)}>First : ``} + {downloads?.paginate?.prev_pages && + downloads?.paginate?.prev_pages?.map((page) => { + return ( + handleSetPageNumber(page)}>{page} + )}) + } + {pageNumber != 1 && < } + Page {pageNumber} + {downloads?.paginate?.last_page && > } + {downloads?.paginate?.next_pages && + downloads?.paginate?.next_pages?.map((page) => { + return ( + handleSetPageNumber(page)}>{page} + )}) + } + {downloads?.paginate?.last_page && + handleSetPageNumber(downloads?.paginate?.last_page)}> Last ({downloads?.paginate?.last_page}) + }
{/* */} diff --git a/tubearchivist/www/src/styles/globals.css b/tubearchivist/www/src/styles/globals.css index d868198..dc51882 100755 --- a/tubearchivist/www/src/styles/globals.css +++ b/tubearchivist/www/src/styles/globals.css @@ -98,6 +98,10 @@ textarea { width: 100%; } +.button-padding { + margin: 0 4px 0 0; +} + button { border-radius: 0; padding: 5px 13px; @@ -326,6 +330,13 @@ button:hover { justify-content: flex-end; } +.view-icons-margin { + /* width: 30px; */ + margin: 5px 10px; + /* cursor: pointer; + filter: var(--img-filter); */ +} + .view-icons img { width: 30px; margin: 5px 10px; @@ -516,6 +527,8 @@ button:hover { .pagination-item { padding: 5px; + margin-right: 4px; + cursor: pointer; border: 1px solid; } diff --git a/tubearchivist/www/src/types/download.ts b/tubearchivist/www/src/types/download.ts index c7d919d..c14eadb 100755 --- a/tubearchivist/www/src/types/download.ts +++ b/tubearchivist/www/src/types/download.ts @@ -1,14 +1,26 @@ export interface Download { data: Datum[]; config: Config; - paginate: boolean; -} - -export interface DownloadResponse { - data: Datum[]; + paginate: Paginate; message: string; } +export interface Paginate { + page_size: number; + page_from: number; + prev_pages: number[]; + current_page: number; + max_hits: boolean; + last_page: number; + next_pages: number[]; + total_hits: number; +} + +// export interface DownloadResponse { +// data: Datum[]; +// message: string; +// } + export interface Config { archive: Archive; default_view: DefaultView;