From 917e73ec4d9f303f826a2d1a37804c5f8da760ab Mon Sep 17 00:00:00 2001 From: simon Date: Mon, 10 Jan 2022 22:51:52 +0700 Subject: [PATCH 1/7] new django api app, implementing basic get views --- tubearchivist/api/README.md | 14 ++++ tubearchivist/api/__init__.py | 0 tubearchivist/api/admin.py | 3 + tubearchivist/api/apps.py | 10 +++ tubearchivist/api/migrations/__init__.py | 0 tubearchivist/api/models.py | 5 ++ tubearchivist/api/serializers.py | 0 tubearchivist/api/tests.py | 3 + tubearchivist/api/urls.py | 33 ++++++++ tubearchivist/api/views.py | 95 ++++++++++++++++++++++++ tubearchivist/config/settings.py | 2 + tubearchivist/config/urls.py | 1 + tubearchivist/requirements.txt | 1 + 13 files changed, 167 insertions(+) create mode 100644 tubearchivist/api/README.md create mode 100644 tubearchivist/api/__init__.py create mode 100644 tubearchivist/api/admin.py create mode 100644 tubearchivist/api/apps.py create mode 100644 tubearchivist/api/migrations/__init__.py create mode 100644 tubearchivist/api/models.py create mode 100644 tubearchivist/api/serializers.py create mode 100644 tubearchivist/api/tests.py create mode 100644 tubearchivist/api/urls.py create mode 100644 tubearchivist/api/views.py diff --git a/tubearchivist/api/README.md b/tubearchivist/api/README.md new file mode 100644 index 0000000..16092fd --- /dev/null +++ b/tubearchivist/api/README.md @@ -0,0 +1,14 @@ +# TubeArchivist API endpoints +Documentation of available API endpoints + +## Videos +/api/video/\/ + +## Channels +/api/channel/\/ + +## Playlists +/api/playlist/\/ + +## Download Queue +/api/download/\/ diff --git a/tubearchivist/api/__init__.py b/tubearchivist/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tubearchivist/api/admin.py b/tubearchivist/api/admin.py new file mode 100644 index 0000000..4fd5490 --- /dev/null +++ b/tubearchivist/api/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin # noqa: F401 + +# Register your models here. diff --git a/tubearchivist/api/apps.py b/tubearchivist/api/apps.py new file mode 100644 index 0000000..9acf5fb --- /dev/null +++ b/tubearchivist/api/apps.py @@ -0,0 +1,10 @@ +"""apps file for api package""" + +from django.apps import AppConfig + + +class ApiConfig(AppConfig): + """app config""" + + default_auto_field = "django.db.models.BigAutoField" + name = "api" diff --git a/tubearchivist/api/migrations/__init__.py b/tubearchivist/api/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tubearchivist/api/models.py b/tubearchivist/api/models.py new file mode 100644 index 0000000..500340a --- /dev/null +++ b/tubearchivist/api/models.py @@ -0,0 +1,5 @@ +"""api models""" + +from django.db import models # noqa: F401 + +# Create your models here. diff --git a/tubearchivist/api/serializers.py b/tubearchivist/api/serializers.py new file mode 100644 index 0000000..e69de29 diff --git a/tubearchivist/api/tests.py b/tubearchivist/api/tests.py new file mode 100644 index 0000000..e55d689 --- /dev/null +++ b/tubearchivist/api/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase # noqa: F401 + +# Create your tests here. diff --git a/tubearchivist/api/urls.py b/tubearchivist/api/urls.py new file mode 100644 index 0000000..7f48576 --- /dev/null +++ b/tubearchivist/api/urls.py @@ -0,0 +1,33 @@ +"""all api urls""" + +from api.views import ( + ChannelApiView, + DownloadApiView, + PlaylistApiView, + VideoApiView, +) +from django.contrib.auth.decorators import login_required +from django.urls import path + +urlpatterns = [ + path( + "video//", + login_required(VideoApiView.as_view()), + name="api-video", + ), + path( + "channel//", + login_required(ChannelApiView.as_view()), + name="api-channel", + ), + path( + "playlist//", + login_required(PlaylistApiView.as_view()), + name="api-playlist", + ), + path( + "download//", + login_required(DownloadApiView.as_view()), + name="api-download", + ), +] diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py new file mode 100644 index 0000000..f250bd3 --- /dev/null +++ b/tubearchivist/api/views.py @@ -0,0 +1,95 @@ +"""all API views""" + +import requests +from home.src.config import AppConfig +from rest_framework.response import Response +from rest_framework.views import APIView + + +class ApiBaseView(APIView): + """base view to inherit from""" + + search_base = False + + def __init__(self): + super().__init__() + self.response = False + self.status_code = False + self.context = False + + def config_builder(self): + """build confic context""" + default_conf = AppConfig().config + self.context = { + "es_url": default_conf["application"]["es_url"], + "es_auth": default_conf["application"]["es_auth"], + } + + def get_document(self, document_id): + """get single document from es""" + es_url = self.context["es_url"] + url = f"{es_url}{self.search_base}{document_id}" + print(url) + response = requests.get(url, auth=self.context["es_auth"]) + self.response = response.json()["_source"] + self.status_code = response.status_code + + +class VideoApiView(ApiBaseView): + """resolves to /api/video// + GET: returns metadata dict of video + """ + + search_base = "/ta_video/_doc/" + + def get(self, request, video_id): + # pylint: disable=unused-argument + """get request""" + self.config_builder() + self.get_document(video_id) + return Response(self.response) + + +class ChannelApiView(ApiBaseView): + """resolves to /api/channel// + GET: returns metadata dict of channel + """ + + search_base = "/ta_channel/_doc/" + + def get(self, request, channel_id): + # pylint: disable=unused-argument + """get request""" + self.config_builder() + self.get_document(channel_id) + return Response(self.response) + + +class PlaylistApiView(ApiBaseView): + """resolves to /api/playlist// + GET: returns metadata dict of playlist + """ + + search_base = "/ta_playlist/_doc/" + + def get(self, request, playlist_id): + # pylint: disable=unused-argument + """get request""" + self.config_builder() + self.get_document(playlist_id) + return Response(self.response) + + +class DownloadApiView(ApiBaseView): + """resolves to /api/download// + GET: returns metadata dict of an item in the download queue + """ + + search_base = "/ta_download/_doc/" + + def get(self, request, video_id): + # pylint: disable=unused-argument + """get request""" + self.config_builder() + self.get_document(video_id) + return Response(self.response) diff --git a/tubearchivist/config/settings.py b/tubearchivist/config/settings.py index 8324c89..21074b3 100644 --- a/tubearchivist/config/settings.py +++ b/tubearchivist/config/settings.py @@ -44,6 +44,8 @@ INSTALLED_APPS = [ "whitenoise.runserver_nostatic", "django.contrib.staticfiles", "django.contrib.humanize", + "rest_framework", + "api", ] MIDDLEWARE = [ diff --git a/tubearchivist/config/urls.py b/tubearchivist/config/urls.py index b857f01..11a1ed7 100644 --- a/tubearchivist/config/urls.py +++ b/tubearchivist/config/urls.py @@ -18,5 +18,6 @@ from django.urls import include, path urlpatterns = [ path("", include("home.urls")), + path("api/", include("api.urls")), path("admin/", admin.site.urls), ] diff --git a/tubearchivist/requirements.txt b/tubearchivist/requirements.txt index a3bc158..3ebbf26 100644 --- a/tubearchivist/requirements.txt +++ b/tubearchivist/requirements.txt @@ -1,6 +1,7 @@ beautifulsoup4==4.10.0 celery==5.2.3 Django==4.0.1 +djangorestframework==3.13.1 Pillow==9.0.0 redis==4.1.0 requests==2.27.1 From 382e89abb73331b96b04df83db5d49901785e1fd Mon Sep 17 00:00:00 2001 From: simon Date: Tue, 11 Jan 2022 14:15:36 +0700 Subject: [PATCH 2/7] implement api token auth --- tubearchivist/api/models.py | 4 +--- tubearchivist/api/urls.py | 9 ++++----- tubearchivist/api/views.py | 7 +++++++ tubearchivist/config/settings.py | 1 + tubearchivist/home/templates/home/settings.html | 4 ++++ tubearchivist/home/views.py | 14 ++++++++++++-- 6 files changed, 29 insertions(+), 10 deletions(-) diff --git a/tubearchivist/api/models.py b/tubearchivist/api/models.py index 500340a..b225e99 100644 --- a/tubearchivist/api/models.py +++ b/tubearchivist/api/models.py @@ -1,5 +1,3 @@ """api models""" -from django.db import models # noqa: F401 - -# Create your models here. +# from django.db import models diff --git a/tubearchivist/api/urls.py b/tubearchivist/api/urls.py index 7f48576..6c471c1 100644 --- a/tubearchivist/api/urls.py +++ b/tubearchivist/api/urls.py @@ -6,28 +6,27 @@ from api.views import ( PlaylistApiView, VideoApiView, ) -from django.contrib.auth.decorators import login_required from django.urls import path urlpatterns = [ path( "video//", - login_required(VideoApiView.as_view()), + VideoApiView.as_view(), name="api-video", ), path( "channel//", - login_required(ChannelApiView.as_view()), + ChannelApiView.as_view(), name="api-channel", ), path( "playlist//", - login_required(PlaylistApiView.as_view()), + PlaylistApiView.as_view(), name="api-playlist", ), path( "download//", - login_required(DownloadApiView.as_view()), + DownloadApiView.as_view(), name="api-download", ), ] diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index f250bd3..3fa5c9f 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -2,6 +2,11 @@ import requests from home.src.config import AppConfig +from rest_framework.authentication import ( + SessionAuthentication, + TokenAuthentication, +) +from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response from rest_framework.views import APIView @@ -9,6 +14,8 @@ from rest_framework.views import APIView class ApiBaseView(APIView): """base view to inherit from""" + authentication_classes = [SessionAuthentication, TokenAuthentication] + permission_classes = [IsAuthenticated] search_base = False def __init__(self): diff --git a/tubearchivist/config/settings.py b/tubearchivist/config/settings.py index 21074b3..cb12ad6 100644 --- a/tubearchivist/config/settings.py +++ b/tubearchivist/config/settings.py @@ -45,6 +45,7 @@ INSTALLED_APPS = [ "django.contrib.staticfiles", "django.contrib.humanize", "rest_framework", + "rest_framework.authtoken", "api", ] diff --git a/tubearchivist/home/templates/home/settings.html b/tubearchivist/home/templates/home/settings.html index 6e728ea..2f35467 100644 --- a/tubearchivist/home/templates/home/settings.html +++ b/tubearchivist/home/templates/home/settings.html @@ -97,6 +97,10 @@

Integrations

+
+

API token:

+

{{ api_token }}

+

Integrate with returnyoutubedislike.com to get dislikes and average ratings back: {{ config.downloads.integrate_ryd }}

Before activating that, make sure you have a scraping sleep interval of at least 3 secs set to avoid ratelimiting issues.
diff --git a/tubearchivist/home/views.py b/tubearchivist/home/views.py index ff48714..d71cea6 100644 --- a/tubearchivist/home/views.py +++ b/tubearchivist/home/views.py @@ -31,6 +31,7 @@ from home.src.index import YoutubePlaylist from home.src.index_management import get_available_backups from home.src.searching import Pagination, SearchHandler from home.tasks import extrac_dl, subscribe_to +from rest_framework.authtoken.models import Token class ArchivistViewConfig(View): @@ -682,8 +683,7 @@ class SettingsView(View): take post request from the form to update settings """ - @staticmethod - def get(request): + def get(self, request): """read and display current settings""" config_handler = AppConfig(request.user.id) colors = config_handler.colors @@ -692,10 +692,12 @@ class SettingsView(View): user_form = UserSettingsForm() app_form = ApplicationSettingsForm() scheduler_form = SchedulerSettingsForm() + token = self.get_token(request) context = { "title": "Settings", "config": config_handler.config, + "api_token": token, "colors": colors, "available_backups": available_backups, "user_form": user_form, @@ -705,6 +707,14 @@ class SettingsView(View): return render(request, "home/settings.html", context) + @staticmethod + def get_token(request): + """get existing or create new token of user""" + # pylint: disable=no-member + token = Token.objects.get_or_create(user=request.user)[0] + print(token) + return token + @staticmethod def post(request): """handle form post to update settings""" From 52a54fbe319c210355dfea193c9af93494049580 Mon Sep 17 00:00:00 2001 From: simon Date: Tue, 11 Jan 2022 15:58:50 +0700 Subject: [PATCH 3/7] add to queue api endpoint --- tubearchivist/api/urls.py | 6 ++++ tubearchivist/api/views.py | 62 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 66 insertions(+), 2 deletions(-) diff --git a/tubearchivist/api/urls.py b/tubearchivist/api/urls.py index 6c471c1..e7c08a8 100644 --- a/tubearchivist/api/urls.py +++ b/tubearchivist/api/urls.py @@ -2,6 +2,7 @@ from api.views import ( ChannelApiView, + DownloadApiListView, DownloadApiView, PlaylistApiView, VideoApiView, @@ -24,6 +25,11 @@ urlpatterns = [ PlaylistApiView.as_view(), name="api-playlist", ), + path( + "download/", + DownloadApiListView.as_view(), + name="api-download-list", + ), path( "download//", DownloadApiView.as_view(), diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index 3fa5c9f..d067507 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -2,6 +2,8 @@ import requests from home.src.config import AppConfig +from home.src.helper import UrlListParser +from home.tasks import extrac_dl from rest_framework.authentication import ( SessionAuthentication, TokenAuthentication, @@ -20,7 +22,7 @@ class ApiBaseView(APIView): def __init__(self): super().__init__() - self.response = False + self.response = {"data": False} self.status_code = False self.context = False @@ -38,7 +40,21 @@ class ApiBaseView(APIView): url = f"{es_url}{self.search_base}{document_id}" print(url) response = requests.get(url, auth=self.context["es_auth"]) - self.response = response.json()["_source"] + self.response["data"] = response.json()["_source"] + self.status_code = response.status_code + + def get_paginate(self): + """add pagination detail to response""" + self.response["paginate"] = False + + def get_document_list(self, data): + """get a list of results""" + es_url = self.context["es_url"] + url = f"{es_url}{self.search_base}" + print(url) + response = requests.get(url, json=data, auth=self.context["es_auth"]) + all_hits = response.json()["hits"]["hits"] + self.response["data"] = [i["_source"] for i in all_hits] self.status_code = response.status_code @@ -100,3 +116,45 @@ class DownloadApiView(ApiBaseView): self.config_builder() self.get_document(video_id) return Response(self.response) + + +class DownloadApiListView(ApiBaseView): + """resolves to /api/download/ + GET: returns latest videos in the download queue + POST: add a list of videos to download queue + """ + + search_base = "/ta_download/_search/" + + def get(self, request): + # pylint: disable=unused-argument + """get request""" + data = {"query": {"match_all": {}}} + self.config_builder() + self.get_document_list(data) + self.get_paginate() + return Response(self.response) + + @staticmethod + def post(request): + """add list of videos to download queue""" + data = request.data + try: + to_add = data["data"] + except KeyError: + message = "missing expected data key" + print(message) + return Response({"message": message}, status=400) + + pending = [i["youtube_id"] for i in to_add if i["status"] == "pending"] + url_str = " ".join(pending) + try: + youtube_ids = UrlListParser(url_str).process_list() + except ValueError: + message = f"failed to parse: {url_str}" + print(message) + return Response({"message": message}, status=400) + + extrac_dl.delay(youtube_ids) + + return Response(data) From 8d5b4ac2429b35ac4ad1b61026a45a17e99937e0 Mon Sep 17 00:00:00 2001 From: simon Date: Tue, 11 Jan 2022 16:16:28 +0700 Subject: [PATCH 4/7] implement api return status code --- tubearchivist/api/views.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index d067507..41df3db 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -40,7 +40,11 @@ class ApiBaseView(APIView): url = f"{es_url}{self.search_base}{document_id}" print(url) response = requests.get(url, auth=self.context["es_auth"]) - self.response["data"] = response.json()["_source"] + try: + self.response["data"] = response.json()["_source"] + except KeyError: + print(f"item not found: {document_id}") + self.response["data"] = False self.status_code = response.status_code def get_paginate(self): @@ -70,7 +74,7 @@ class VideoApiView(ApiBaseView): """get request""" self.config_builder() self.get_document(video_id) - return Response(self.response) + return Response(self.response, status=self.status_code) class ChannelApiView(ApiBaseView): @@ -85,7 +89,7 @@ class ChannelApiView(ApiBaseView): """get request""" self.config_builder() self.get_document(channel_id) - return Response(self.response) + return Response(self.response, status=self.status_code) class PlaylistApiView(ApiBaseView): @@ -100,7 +104,7 @@ class PlaylistApiView(ApiBaseView): """get request""" self.config_builder() self.get_document(playlist_id) - return Response(self.response) + return Response(self.response, status=self.status_code) class DownloadApiView(ApiBaseView): @@ -115,7 +119,7 @@ class DownloadApiView(ApiBaseView): """get request""" self.config_builder() self.get_document(video_id) - return Response(self.response) + return Response(self.response, status=self.status_code) class DownloadApiListView(ApiBaseView): From 50006e423c2ecb2766f8c5e1270da32104221578 Mon Sep 17 00:00:00 2001 From: simon Date: Tue, 11 Jan 2022 16:53:02 +0700 Subject: [PATCH 5/7] implement channel list and subscribe api --- tubearchivist/api/urls.py | 6 ++++++ tubearchivist/api/views.py | 38 +++++++++++++++++++++++++++++++++++++- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/tubearchivist/api/urls.py b/tubearchivist/api/urls.py index e7c08a8..d39dc30 100644 --- a/tubearchivist/api/urls.py +++ b/tubearchivist/api/urls.py @@ -1,6 +1,7 @@ """all api urls""" from api.views import ( + ChannelApiListView, ChannelApiView, DownloadApiListView, DownloadApiView, @@ -15,6 +16,11 @@ urlpatterns = [ VideoApiView.as_view(), name="api-video", ), + path( + "channel/", + ChannelApiListView.as_view(), + name="api-channel-list", + ), path( "channel//", ChannelApiView.as_view(), diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index 41df3db..ccbc22f 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -3,7 +3,7 @@ import requests from home.src.config import AppConfig from home.src.helper import UrlListParser -from home.tasks import extrac_dl +from home.tasks import extrac_dl, subscribe_to from rest_framework.authentication import ( SessionAuthentication, TokenAuthentication, @@ -92,6 +92,42 @@ class ChannelApiView(ApiBaseView): return Response(self.response, status=self.status_code) +class ChannelApiListView(ApiBaseView): + """resolves to /api/channel/ + GET: returns list of channels + POST: edit a list of channels + """ + + search_base = "/ta_channel/_search/" + + def get(self, request): + # pylint: disable=unused-argument + """get request""" + data = {"query": {"match_all": {}}} + self.config_builder() + self.get_document_list(data) + self.get_paginate() + + return Response(self.response) + + @staticmethod + def post(request): + """subscribe to list of channels""" + data = request.data + try: + to_add = data["data"] + except KeyError: + message = "missing expected data key" + print(message) + return Response({"message": message}, status=400) + + pending = [i["channel_id"] for i in to_add if i["channel_subscribed"]] + url_str = " ".join(pending) + subscribe_to.delay(url_str) + + return Response(data) + + class PlaylistApiView(ApiBaseView): """resolves to /api/playlist// GET: returns metadata dict of playlist From b28773b3a0a368a96715d8fbc0da33de72adccef Mon Sep 17 00:00:00 2001 From: simon Date: Tue, 11 Jan 2022 16:53:58 +0700 Subject: [PATCH 6/7] add exmples, auth docs, and list endpoints --- tubearchivist/api/README.md | 54 ++++++++++++++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 4 deletions(-) diff --git a/tubearchivist/api/README.md b/tubearchivist/api/README.md index 16092fd..e838002 100644 --- a/tubearchivist/api/README.md +++ b/tubearchivist/api/README.md @@ -1,14 +1,60 @@ -# TubeArchivist API endpoints -Documentation of available API endpoints +# TubeArchivist API +Documentation of available API endpoints. +**Note: This will change!** + +## Authentication +API token will get automatically created, accessible on the settings page. Token needs to be passed as an authorization header with every request. Additionally session based authentication is enabled too: When you are logged in to your TubeArchivist instance, you'll have access to the api in the browser for testing. + +Curl example: +```shell +curl -v /api/video// \ + -H "Authorization: Token xxxxxxxxxx" +``` + +Python requests example: +```python +import requests + +url = "/api/video//" +headers = {"Authorization": "Token xxxxxxxxxx"} +response = requests.get(url, headers=headers) +``` ## Videos /api/video/\/ -## Channels +## Channel List View +/api/channel/ + +### Subscribe to a list of channels +POST /api/channel +```json +{ + "data": [ + {"channel_id": "UC9-y-6csu5WGm29I7JiwpnA", "channel_subscribed": true} + ] +} +``` + +## Channel Item View /api/channel/\/ ## Playlists /api/playlist/\/ -## Download Queue +## Download Queue List View +/api/download/ + +### Add list of videos to download queue + +POST /api/download/ +```json +{ + "data": [ + {"youtube_id": "NYj3DnI81AQ", "status": "pending"} + ] +} +``` + +## Download Queue Item View /api/download/\/ From c4a547f4073e6c7ec2126af22131a1964f1f1c37 Mon Sep 17 00:00:00 2001 From: simon Date: Tue, 11 Jan 2022 17:05:04 +0700 Subject: [PATCH 7/7] cleanup and typos --- tubearchivist/api/README.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tubearchivist/api/README.md b/tubearchivist/api/README.md index e838002..73dd4fc 100644 --- a/tubearchivist/api/README.md +++ b/tubearchivist/api/README.md @@ -1,9 +1,9 @@ # TubeArchivist API Documentation of available API endpoints. -**Note: This will change!** +**Note: This is very early alpha and will change!** ## Authentication -API token will get automatically created, accessible on the settings page. Token needs to be passed as an authorization header with every request. Additionally session based authentication is enabled too: When you are logged in to your TubeArchivist instance, you'll have access to the api in the browser for testing. +API token will get automatically created, accessible on the settings page. Token needs to be passed as an authorization header with every request. Additionally session based authentication is enabled too: When you are logged into your TubeArchivist instance, you'll have access to the api in the browser for testing. Curl example: ```shell @@ -20,14 +20,14 @@ headers = {"Authorization": "Token xxxxxxxxxx"} response = requests.get(url, headers=headers) ``` -## Videos +## Video Item View /api/video/\/ ## Channel List View /api/channel/ ### Subscribe to a list of channels -POST /api/channel +POST /api/channel/ ```json { "data": [ @@ -39,14 +39,13 @@ POST /api/channel ## Channel Item View /api/channel/\/ -## Playlists +## Playlists Item View /api/playlist/\/ ## Download Queue List View /api/download/ ### Add list of videos to download queue - POST /api/download/ ```json {