manual import fix, #build
Changed: - fixed manual import errors, #311 #312 - fixed channel extraction to catch all alerts - add LDAP disable cert check - downgrade django, fix for DJANGO_DEBUG env var
This commit is contained in:
commit
265a53ace3
|
|
@ -15,7 +15,7 @@ body:
|
|||
options:
|
||||
- label: I have read through the [wiki](https://github.com/tubearchivist/tubearchivist/wiki).
|
||||
required: true
|
||||
- label: I understand the [scope](https://github.com/tubearchivist/tubearchivist/wiki/FAQ) of this project and am aware of the [known limitations](https://github.com/tubearchivist/tubearchivist#known-limitations).
|
||||
- label: I understand the [scope](https://github.com/tubearchivist/tubearchivist/wiki/FAQ) of this project and am aware of the [known limitations](https://github.com/tubearchivist/tubearchivist#known-limitations) and my idea is not already on the [roadmap](https://github.com/tubearchivist/tubearchivist#roadmap).
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
|
|
|
|||
|
|
@ -72,7 +72,7 @@ The main Python application that displays and serves your video collection, buil
|
|||
- And another volume to save application data at **/cache**.
|
||||
- The environment variables `ES_URL` and `REDIS_HOST` are needed to tell Tube Archivist where Elasticsearch and Redis respectively are located.
|
||||
- The environment variables `HOST_UID` and `HOST_GID` allows Tube Archivist to `chown` the video files to the main host system user instead of the container user. Those two variables are optional, not setting them will disable that functionality. That might be needed if the underlying filesystem doesn't support `chown` like *NFS*.
|
||||
- Set the environment variable `TA_HOST` to match with the system running Tube Archivist. This can be a domain like *example.com*, a subdomain like *ta.example.com* or an IP address like *192.168.1.20*, add without the protocol and without the port. You can add multiple hostnames separated with a space.
|
||||
- Set the environment variable `TA_HOST` to match with the system running Tube Archivist. This can be a domain like *example.com*, a subdomain like *ta.example.com* or an IP address like *192.168.1.20*, add without the protocol and without the port. You can add multiple hostnames separated with a space. Any wrong configurations here will result in a `Bad Request (400)` response.
|
||||
- Change the environment variables `TA_USERNAME` and `TA_PASSWORD` to create the initial credentials.
|
||||
- `ELASTIC_PASSWORD` is for the password for Elasticsearch. The environment variable `ELASTIC_USER` is optional, should you want to change the username from the default *elastic*.
|
||||
- For the scheduler to know what time it is, set your timezone with the `TZ` environment variable, defaults to *UTC*.
|
||||
|
|
|
|||
|
|
@ -124,7 +124,7 @@ class ThumbManager(ThumbManagerBase):
|
|||
|
||||
img_raw.convert("RGB").save(thumb_path)
|
||||
|
||||
def vid_thumb_path(self, absolute=False):
|
||||
def vid_thumb_path(self, absolute=False, create_folder=False):
|
||||
"""build expected path for video thumbnail from youtube_id"""
|
||||
folder_name = self.item_id[0].lower()
|
||||
folder_path = os.path.join("videos", folder_name)
|
||||
|
|
@ -132,6 +132,10 @@ class ThumbManager(ThumbManagerBase):
|
|||
if absolute:
|
||||
thumb_path = os.path.join(self.CACHE_DIR, thumb_path)
|
||||
|
||||
if create_folder:
|
||||
folder_path = os.path.join(self.CACHE_DIR, folder_path)
|
||||
os.makedirs(folder_path, exist_ok=True)
|
||||
|
||||
return thumb_path
|
||||
|
||||
def download_channel_art(self, urls, skip_existing=False):
|
||||
|
|
|
|||
|
|
@ -73,13 +73,14 @@ class ChannelScraper:
|
|||
|
||||
def _is_deactivated(self):
|
||||
"""check if channel is deactivated"""
|
||||
alert_text = "This channel does not exist."
|
||||
alerts = self.yt_json.get("alerts")
|
||||
if alerts and alert_text in str(alerts):
|
||||
print(f"{self.channel_id}: {alert_text}")
|
||||
return True
|
||||
if not alerts:
|
||||
return False
|
||||
|
||||
return False
|
||||
for alert in alerts:
|
||||
alert_text = alert["alertRenderer"]["text"]["simpleText"]
|
||||
print(f"{self.channel_id}: failed to extract, {alert_text}")
|
||||
return True
|
||||
|
||||
def _parse_channel_main(self):
|
||||
"""extract maintab values from scraped channel json data"""
|
||||
|
|
@ -226,8 +227,8 @@ class YoutubeChannel(YouTubeItem):
|
|||
|
||||
self.json_data.update(
|
||||
{
|
||||
"channel_subs": content["channel_follower_count"],
|
||||
"channel_description": content["description"],
|
||||
"channel_subs": content.get("channel_follower_count", 0),
|
||||
"channel_description": content.get("description", False),
|
||||
}
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -217,15 +217,14 @@ class ImportFolderScanner:
|
|||
last_base = False
|
||||
|
||||
for file_path in all_files:
|
||||
base_name_raw, ext = os.path.splitext(file_path)
|
||||
base_name, _ = os.path.splitext(base_name_raw)
|
||||
|
||||
base_name, ext = self._detect_base_name(file_path)
|
||||
key, file_path = self._detect_type(file_path, ext)
|
||||
if not key or not file_path:
|
||||
continue
|
||||
|
||||
if base_name != last_base:
|
||||
if last_base:
|
||||
print(f"manual import: {current_video}")
|
||||
self.to_import.append(current_video)
|
||||
|
||||
current_video = self._get_template()
|
||||
|
|
@ -237,8 +236,21 @@ class ImportFolderScanner:
|
|||
current_video[key] = file_path
|
||||
|
||||
if current_video.get("media"):
|
||||
print(f"manual import: {current_video}")
|
||||
self.to_import.append(current_video)
|
||||
|
||||
def _detect_base_name(self, file_path):
|
||||
"""extract base_name and ext for matching"""
|
||||
base_name_raw, ext = os.path.splitext(file_path)
|
||||
base_name, ext2 = os.path.splitext(base_name_raw)
|
||||
|
||||
if ext2:
|
||||
if ISO639Utils.short2long(ext2.strip(".")) or ext2 == ".info":
|
||||
# valid secondary extension
|
||||
return base_name, ext
|
||||
|
||||
return base_name_raw, ext
|
||||
|
||||
def _detect_type(self, file_path, ext):
|
||||
"""detect metadata type for file"""
|
||||
|
||||
|
|
@ -260,12 +272,12 @@ class ImportFolderScanner:
|
|||
self._convert_thumb(current_video)
|
||||
self._get_subtitles(current_video)
|
||||
self._convert_video(current_video)
|
||||
print(f"manual import: {current_video}")
|
||||
|
||||
ManualImport(current_video, self.CONFIG).run()
|
||||
|
||||
def _detect_youtube_id(self, current_video):
|
||||
"""find video id from filename or json"""
|
||||
print(current_video)
|
||||
youtube_id = self._extract_id_from_filename(current_video["media"])
|
||||
if youtube_id:
|
||||
current_video["video_id"] = youtube_id
|
||||
|
|
@ -276,7 +288,6 @@ class ImportFolderScanner:
|
|||
current_video["video_id"] = youtube_id
|
||||
return
|
||||
|
||||
print(current_video["media"])
|
||||
raise ValueError("failed to find video id")
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -516,7 +527,8 @@ class ManualImport:
|
|||
|
||||
if video.offline_import and self.current_video["thumb"]:
|
||||
old_path = self.current_video["thumb"]
|
||||
new_path = ThumbManager(video_id).vid_thumb_path(absolute=True)
|
||||
thumbs = ThumbManager(video_id)
|
||||
new_path = thumbs.vid_thumb_path(absolute=True, create_folder=True)
|
||||
shutil.move(old_path, new_path, copy_function=shutil.copyfile)
|
||||
else:
|
||||
url = video.json_data["vid_thumb_url"]
|
||||
|
|
@ -555,11 +567,13 @@ class ManualImport:
|
|||
|
||||
def _cleanup(self, json_data):
|
||||
"""cleanup leftover files"""
|
||||
if os.path.exists(self.current_video["metadata"]):
|
||||
os.remove(self.current_video["metadata"])
|
||||
meta_data = self.current_video["metadata"]
|
||||
if meta_data and os.path.exists(meta_data):
|
||||
os.remove(meta_data)
|
||||
|
||||
if os.path.exists(self.current_video["thumb"]):
|
||||
os.remove(self.current_video["thumb"])
|
||||
thumb = self.current_video["thumb"]
|
||||
if thumb and os.path.exists(thumb):
|
||||
os.remove(thumb)
|
||||
|
||||
for subtitle_file in self.current_video["subtitle"]:
|
||||
if os.path.exists(subtitle_file):
|
||||
|
|
|
|||
|
|
@ -10,4 +10,4 @@ requests==2.28.1
|
|||
ryd-client==0.0.6
|
||||
uWSGI==2.0.20
|
||||
whitenoise==6.2.0
|
||||
yt_dlp==2022.8.19
|
||||
yt_dlp==2022.9.1
|
||||
|
|
|
|||
Loading…
Reference in New Issue