From 93822fc43cfcf21bb818557f3c631bbfb8ecd2cb Mon Sep 17 00:00:00 2001 From: n8detar Date: Sun, 17 Apr 2022 12:47:18 -0700 Subject: [PATCH 01/31] API changes --- tubearchivist/www/src/lib/getDownloads.ts | 10 ++-- tubearchivist/www/src/pages/download.tsx | 63 ++++++++--------------- 2 files changed, 28 insertions(+), 45 deletions(-) mode change 100644 => 100755 tubearchivist/www/src/lib/getDownloads.ts diff --git a/tubearchivist/www/src/lib/getDownloads.ts b/tubearchivist/www/src/lib/getDownloads.ts old mode 100644 new mode 100755 index e610a90..560185d --- a/tubearchivist/www/src/lib/getDownloads.ts +++ b/tubearchivist/www/src/lib/getDownloads.ts @@ -1,9 +1,11 @@ import { Download } from "../types/download"; import { DownloadResponse } from "../types/download"; -import { TA_BASE_URL } from "./constants"; +import { getTAUrl } from "./constants"; -export const getDownloads = async (token: string): Promise => { - const response = await fetch(`${TA_BASE_URL}/api/download/`, { +const TA_BASE_URL = getTAUrl(); + +export const getDownloads = async (token: string, ignoredStatus: boolean): Promise => { + const response = await fetch(`${TA_BASE_URL.server}/api/download/?filter=${ignoredStatus ? 'ignore' : 'pending'}`, { headers: { Accept: "application/json", "Content-Type": "application/json", @@ -24,7 +26,7 @@ export const sendDownloads = async (token: string, input: string): Promise { } await queryClient.prefetchQuery(["downloads", session.ta_token.token], () => - getDownloads(session.ta_token.token) + getDownloads(session.ta_token.token, false) ); return { @@ -51,16 +52,17 @@ const Download: NextPage = () => { data: downloads, error, isLoading, + refetch, } = useQuery( ["downloads", session.ta_token.token], - () => getDownloads(session.ta_token.token), + () => getDownloads(session.ta_token.token, ignoredStatus), { enabled: !!session?.ta_token?.token, + refetchInterval: 1500, + refetchIntervalInBackground: false, } ); - var count = 0; - const [viewStyle, setViewStyle] = useState(downloads?.config?.default_view?.downloads); const [ignoredStatus, setIgnoredStatus] = useState(false); const [formHidden, setFormHidden] = useState(true); @@ -71,6 +73,7 @@ const Download: NextPage = () => { const handleSetIgnoredStatus = (selectedIgnoredStatus: IgnoredStatus) => { setIgnoredStatus(selectedIgnoredStatus); + refetch(); }; const handleSetFormHidden = (selectedFormHidden: FormHidden) => { @@ -142,31 +145,17 @@ const Download: NextPage = () => {
Show only ignored videos: - {ignoredStatus && -
- handleSetIgnoredStatus(false)} - type="checkbox" - checked - /> - -
- } - {!ignoredStatus && -
- handleSetIgnoredStatus(true)} - type="checkbox" - /> - -
- } +
+ handleSetIgnoredStatus(!ignoredStatus)} + type="checkbox" + checked={ignoredStatus} + /> + +
{
} - {downloads?.data?.forEach((video) => { - if ((video?.status == "ignore" && ignoredStatus) || (video?.status == "pending" && !ignoredStatus)) { - count++; - } - })} -

Total videos: {count} {!count &&

No videos queued for download. Press rescan subscriptions to check if there are any new videos.

}

+

Total videos: {downloads?.data?.length} {!downloads?.data?.length &&

No videos queued for download. Press rescan subscriptions to check if there are any new videos.

}

- {downloads.data && + {downloads?.data && downloads?.data?.map((video) => { - count++; - if ((video?.status == "ignore" && ignoredStatus) || (video?.status == "pending" && !ignoredStatus)) { return (
- video_thumb + video_thumb {ignoredStatus && ignored} {/* {% if show_ignored_only %} */} {/* ignored */} @@ -257,7 +239,6 @@ const Download: NextPage = () => {
); - } }) } {/* {% if results %} */} From 46c6e7e9252688a5af13a748b90f4ec55ceb3e0f Mon Sep 17 00:00:00 2001 From: n8detar Date: Sun, 17 Apr 2022 16:32:10 -0700 Subject: [PATCH 02/31] Use filter API for download queue --- tubearchivist/www/src/lib/getDownloads.ts | 4 ++-- tubearchivist/www/src/pages/download.tsx | 19 ++++++++++++------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/tubearchivist/www/src/lib/getDownloads.ts b/tubearchivist/www/src/lib/getDownloads.ts index 560185d..04dde39 100755 --- a/tubearchivist/www/src/lib/getDownloads.ts +++ b/tubearchivist/www/src/lib/getDownloads.ts @@ -4,8 +4,8 @@ import { getTAUrl } from "./constants"; const TA_BASE_URL = getTAUrl(); -export const getDownloads = async (token: string, ignoredStatus: boolean): Promise => { - const response = await fetch(`${TA_BASE_URL.server}/api/download/?filter=${ignoredStatus ? 'ignore' : 'pending'}`, { +export const getDownloads = async (token: string, filter: boolean): Promise => { + const response = await fetch(`${TA_BASE_URL.server}/api/download/?filter=${filter ? 'ignore' : 'pending'}`, { headers: { Accept: "application/json", "Content-Type": "application/json", diff --git a/tubearchivist/www/src/pages/download.tsx b/tubearchivist/www/src/pages/download.tsx index 2776fe7..dfe87d1 100755 --- a/tubearchivist/www/src/pages/download.tsx +++ b/tubearchivist/www/src/pages/download.tsx @@ -34,7 +34,7 @@ export const getServerSideProps: GetServerSideProps = async (context) => { }; } - await queryClient.prefetchQuery(["downloads", session.ta_token.token], () => + await queryClient.prefetchQuery(["downloads", session.ta_token.token, false], () => getDownloads(session.ta_token.token, false) ); @@ -48,33 +48,38 @@ export const getServerSideProps: GetServerSideProps = async (context) => { const Download: NextPage = () => { const { data: session } = useSession(); + + const [ignoredStatus, setIgnoredStatus] = useState(false); + const [formHidden, setFormHidden] = useState(true); + const { data: downloads, error, isLoading, refetch, } = useQuery( - ["downloads", session.ta_token.token], + ["downloads", session.ta_token.token, ignoredStatus], () => getDownloads(session.ta_token.token, ignoredStatus), { enabled: !!session?.ta_token?.token, - refetchInterval: 1500, - refetchIntervalInBackground: false, + // refetchInterval: 1500, + // refetchIntervalInBackground: false, } ); const [viewStyle, setViewStyle] = useState(downloads?.config?.default_view?.downloads); - const [ignoredStatus, setIgnoredStatus] = useState(false); - const [formHidden, setFormHidden] = useState(true); + const handleSetViewstyle = (selectedViewStyle: ViewStyle) => { setViewStyle(selectedViewStyle); }; - + const handleSetIgnoredStatus = (selectedIgnoredStatus: IgnoredStatus) => { + // ignoredStatus = !ignoredStatus; setIgnoredStatus(selectedIgnoredStatus); refetch(); }; + const handleSetFormHidden = (selectedFormHidden: FormHidden) => { setFormHidden(selectedFormHidden); From 3147df20da96efd01e23bdd27300176bdeba2177 Mon Sep 17 00:00:00 2001 From: simon Date: Mon, 18 Apr 2022 11:52:13 +0700 Subject: [PATCH 03/31] skip subtitle segments without duration, take 2 --- tubearchivist/home/src/index/video.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tubearchivist/home/src/index/video.py b/tubearchivist/home/src/index/video.py index 969d716..290e0ce 100644 --- a/tubearchivist/home/src/index/video.py +++ b/tubearchivist/home/src/index/video.py @@ -195,6 +195,11 @@ class SubtitleParser: if flatten: # fix overlapping retiming issue + if "dDurationMs" not in flatten[-1]: + # some events won't have a duration + print(f"failed to parse event without duration: {event}") + continue + last_end = flatten[-1]["tStartMs"] + flatten[-1]["dDurationMs"] if event["tStartMs"] < last_end: joined = flatten[-1]["segs"][0]["utf8"] + "\n" + text From a5c50fd6902032270dee5fb073202333ab663371 Mon Sep 17 00:00:00 2001 From: n8detar Date: Mon, 18 Apr 2022 09:00:53 -0700 Subject: [PATCH 04/31] Added invalid input message. --- tubearchivist/www/src/lib/getDownloads.ts | 3 ++- tubearchivist/www/src/pages/download.tsx | 23 ++++++++++++++++------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/tubearchivist/www/src/lib/getDownloads.ts b/tubearchivist/www/src/lib/getDownloads.ts index 04dde39..546ac95 100755 --- a/tubearchivist/www/src/lib/getDownloads.ts +++ b/tubearchivist/www/src/lib/getDownloads.ts @@ -37,7 +37,8 @@ export const sendDownloads = async (token: string, input: string): Promise { @@ -51,6 +52,7 @@ const Download: NextPage = () => { const [ignoredStatus, setIgnoredStatus] = useState(false); const [formHidden, setFormHidden] = useState(true); + const { data: downloads, @@ -62,13 +64,13 @@ const Download: NextPage = () => { () => getDownloads(session.ta_token.token, ignoredStatus), { enabled: !!session?.ta_token?.token, - // refetchInterval: 1500, - // refetchIntervalInBackground: false, + refetchInterval: 1500, + refetchIntervalInBackground: false, } ); const [viewStyle, setViewStyle] = useState(downloads?.config?.default_view?.downloads); - + const [errorMessage, setErrorMessage] = useState(false); const handleSetViewstyle = (selectedViewStyle: ViewStyle) => { setViewStyle(selectedViewStyle); @@ -85,10 +87,14 @@ const Download: NextPage = () => { setFormHidden(selectedFormHidden); }; + const handleSetErrorMessage = (selectedErrorMessage: ErrorMessage) => { + setErrorMessage(selectedErrorMessage); + }; + const addToDownloadQueue = event => { event.preventDefault(); - sendDownloads(session.ta_token.token, event.target.vid_url.value); - handleSetFormHidden(true); + sendDownloads(session.ta_token.token, event.target.vid_url.value).then((response) => !response.message ? handleSetErrorMessage(false) : handleSetErrorMessage(true)); + errorMessage ? handleSetFormHidden(false) : handleSetFormHidden(true); } return ( @@ -141,6 +147,9 @@ const Download: NextPage = () => {