2021-09-05 17:10:14 +00:00
|
|
|
"""
|
|
|
|
Functionality:
|
|
|
|
- initiate celery app
|
|
|
|
- collect tasks
|
2021-11-29 08:49:45 +00:00
|
|
|
- user config changes won't get applied here
|
|
|
|
because tasks are initiated at application start
|
2021-09-05 17:10:14 +00:00
|
|
|
"""
|
|
|
|
|
|
|
|
import os
|
|
|
|
|
2023-03-13 03:09:48 +00:00
|
|
|
from celery import Celery, Task, shared_task
|
2022-01-22 15:13:37 +00:00
|
|
|
from home.src.download.queue import PendingList
|
|
|
|
from home.src.download.subscriptions import (
|
2023-03-02 05:30:48 +00:00
|
|
|
SubscriptionHandler,
|
2023-03-01 12:12:27 +00:00
|
|
|
SubscriptionScanner,
|
2021-11-18 09:00:55 +00:00
|
|
|
)
|
2022-08-10 14:03:54 +00:00
|
|
|
from home.src.download.thumbnails import ThumbFilesystem, ThumbValidator
|
2022-01-22 15:13:37 +00:00
|
|
|
from home.src.download.yt_dlp_handler import VideoDownloader
|
2022-11-27 08:41:59 +00:00
|
|
|
from home.src.es.backup import ElasticBackup
|
|
|
|
from home.src.es.index_setup import ElasitIndexWrap
|
2022-01-22 15:13:37 +00:00
|
|
|
from home.src.index.channel import YoutubeChannel
|
2023-03-16 06:18:26 +00:00
|
|
|
from home.src.index.filesystem import Filesystem
|
2023-03-16 03:59:47 +00:00
|
|
|
from home.src.index.manual import ImportFolderScanner
|
2022-12-11 08:56:44 +00:00
|
|
|
from home.src.index.reindex import Reindex, ReindexManual, ReindexOutdated
|
2022-12-21 07:24:24 +00:00
|
|
|
from home.src.ta.config import AppConfig, ReleaseVersion, ScheduleBuilder
|
2023-01-06 08:37:55 +00:00
|
|
|
from home.src.ta.helper import clear_dl_cache
|
2022-01-22 15:13:37 +00:00
|
|
|
from home.src.ta.ta_redis import RedisArchivist, RedisQueue
|
2023-03-01 12:12:27 +00:00
|
|
|
from home.src.ta.task_manager import TaskManager
|
2021-09-05 17:10:14 +00:00
|
|
|
|
|
|
|
CONFIG = AppConfig().config
|
2021-09-30 11:03:23 +00:00
|
|
|
REDIS_HOST = os.environ.get("REDIS_HOST")
|
2021-11-29 08:49:45 +00:00
|
|
|
REDIS_PORT = os.environ.get("REDIS_PORT") or 6379
|
2021-09-05 17:10:14 +00:00
|
|
|
|
2021-12-02 08:54:29 +00:00
|
|
|
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
|
2023-03-01 08:31:17 +00:00
|
|
|
app = Celery(
|
|
|
|
"tasks",
|
|
|
|
broker=f"redis://{REDIS_HOST}:{REDIS_PORT}",
|
|
|
|
backend=f"redis://{REDIS_HOST}:{REDIS_PORT}",
|
|
|
|
result_extended=True,
|
|
|
|
)
|
2021-10-27 11:07:35 +00:00
|
|
|
app.config_from_object("django.conf:settings", namespace="ta:")
|
2021-09-05 17:10:14 +00:00
|
|
|
app.autodiscover_tasks()
|
2021-12-03 05:00:40 +00:00
|
|
|
app.conf.timezone = os.environ.get("TZ") or "UTC"
|
2021-09-05 17:10:14 +00:00
|
|
|
|
|
|
|
|
2023-03-13 03:09:48 +00:00
|
|
|
class BaseTask(Task):
|
|
|
|
"""base class to inherit each class from"""
|
|
|
|
|
|
|
|
TASK_CONFIG = {
|
|
|
|
"update_subscribed": {
|
|
|
|
"title": "Rescan your Subscriptions",
|
|
|
|
"group": "message:download:scan",
|
|
|
|
},
|
|
|
|
"download_pending": {
|
|
|
|
"title": "Downloading",
|
|
|
|
"group": "message:download:run",
|
|
|
|
},
|
|
|
|
"extract_download": {
|
|
|
|
"title": "Add to download queue",
|
|
|
|
"group": "message:download:add",
|
|
|
|
},
|
|
|
|
"subscribe_to": {
|
|
|
|
"title": "Add Subscription",
|
|
|
|
"group": "message:download:subscribe",
|
|
|
|
},
|
|
|
|
"check_reindex": {
|
2023-03-15 09:59:20 +00:00
|
|
|
"title": "Reindex Documents",
|
2023-03-13 03:09:48 +00:00
|
|
|
"group": "message:settings:reindex",
|
|
|
|
},
|
|
|
|
"manual_import": {
|
|
|
|
"title": "Manual video import",
|
|
|
|
"group": "message:settings:import",
|
|
|
|
},
|
|
|
|
"run_backup": {
|
|
|
|
"title": "Index Backup",
|
|
|
|
"group": "message:settings:backup",
|
|
|
|
},
|
|
|
|
"restore_backup": {
|
|
|
|
"title": "Restore Backup",
|
|
|
|
"group": "message:settings:restore",
|
|
|
|
},
|
|
|
|
"rescan_filesystem": {
|
|
|
|
"title": "Rescan your Filesystem",
|
|
|
|
"group": "message:settings:filesystemscan",
|
|
|
|
},
|
|
|
|
"thumbnail_check": {
|
|
|
|
"title": "Check your Thumbnails",
|
|
|
|
"group": "message:settings:thumbnailcheck",
|
|
|
|
},
|
|
|
|
"resync_thumbs": {
|
|
|
|
"title": "Sync Thumbnails to Media Files",
|
|
|
|
"group": "message:settings:thumbnailsync",
|
|
|
|
},
|
|
|
|
"index_playlists": {
|
|
|
|
"title": "Index Channel Playlist",
|
|
|
|
"group": "message:channel:indexplaylist",
|
|
|
|
},
|
|
|
|
}
|
|
|
|
|
|
|
|
def on_failure(self, exc, task_id, args, kwargs, einfo):
|
|
|
|
"""callback for task failure"""
|
|
|
|
print(f"{task_id} Failed callback")
|
|
|
|
|
|
|
|
def on_success(self, retval, task_id, args, kwargs):
|
|
|
|
"""callback task completed successfully"""
|
|
|
|
print(f"{task_id} success callback")
|
2023-03-14 08:42:42 +00:00
|
|
|
message, key = self._build_message()
|
|
|
|
message.update({"messages": ["Task completed successfully"]})
|
2023-03-13 03:09:48 +00:00
|
|
|
RedisArchivist().set_message(key, message, expire=5)
|
|
|
|
|
|
|
|
def before_start(self, task_id, args, kwargs):
|
|
|
|
"""callback before initiating task"""
|
|
|
|
print(f"{self.name} create callback")
|
2023-03-14 08:42:42 +00:00
|
|
|
message, key = self._build_message()
|
|
|
|
message.update({"messages": ["New task received."]})
|
2023-03-13 03:09:48 +00:00
|
|
|
RedisArchivist().set_message(key, message)
|
|
|
|
|
2023-03-15 06:28:19 +00:00
|
|
|
def send_progress(self, message_lines, progress=False, title=False):
|
2023-03-14 08:42:42 +00:00
|
|
|
"""send progress message"""
|
|
|
|
message, key = self._build_message()
|
|
|
|
message.update(
|
|
|
|
{
|
|
|
|
"messages": message_lines,
|
|
|
|
"progress": progress,
|
|
|
|
}
|
|
|
|
)
|
2023-03-15 06:28:19 +00:00
|
|
|
if title:
|
|
|
|
message["title"] = title
|
|
|
|
|
2023-03-14 08:42:42 +00:00
|
|
|
RedisArchivist().set_message(key, message)
|
2023-03-13 03:09:48 +00:00
|
|
|
|
2023-03-14 08:42:42 +00:00
|
|
|
def _build_message(self, level="info"):
|
2023-03-13 03:09:48 +00:00
|
|
|
"""build message dict"""
|
2023-03-14 08:42:42 +00:00
|
|
|
task_id = self.request.id
|
2023-03-13 03:09:48 +00:00
|
|
|
config = self.TASK_CONFIG.get(self.name)
|
|
|
|
message = {
|
|
|
|
"status": config.get("group"),
|
|
|
|
"title": config.get("title"),
|
|
|
|
"level": level,
|
|
|
|
"id": task_id,
|
|
|
|
}
|
|
|
|
key = f"{config.get('group')}:{task_id.split('-')[0]}"
|
|
|
|
return message, key
|
|
|
|
|
|
|
|
|
2023-03-14 08:42:42 +00:00
|
|
|
@shared_task(name="update_subscribed", bind=True, base=BaseTask)
|
2023-03-01 12:12:27 +00:00
|
|
|
def update_subscribed(self):
|
2021-09-21 09:25:22 +00:00
|
|
|
"""look for missing videos and add to pending"""
|
2023-03-01 12:12:27 +00:00
|
|
|
manager = TaskManager()
|
|
|
|
if manager.is_pending(self):
|
|
|
|
print(f"[task][{self.name}] rescan already running")
|
|
|
|
message = {
|
|
|
|
"status": "message:rescan",
|
|
|
|
"level": "error",
|
|
|
|
"title": "Rescanning channels and playlists.",
|
|
|
|
"message": "Rescan already in progress.",
|
|
|
|
}
|
|
|
|
RedisArchivist().set_message("message:rescan", message, expire=True)
|
|
|
|
return
|
|
|
|
|
|
|
|
manager.init(self)
|
2023-03-14 09:40:05 +00:00
|
|
|
missing_videos = SubscriptionScanner(task=self).scan()
|
|
|
|
if missing_videos:
|
|
|
|
print(missing_videos)
|
|
|
|
extrac_dl.delay(missing_videos)
|
2023-03-01 12:12:27 +00:00
|
|
|
|
|
|
|
|
2023-03-15 06:28:19 +00:00
|
|
|
@shared_task(name="download_pending", bind=True, base=BaseTask)
|
2023-03-02 07:17:03 +00:00
|
|
|
def download_pending(self, from_queue=True):
|
2021-09-21 09:25:22 +00:00
|
|
|
"""download latest pending videos"""
|
2023-03-01 12:12:27 +00:00
|
|
|
manager = TaskManager()
|
|
|
|
if manager.is_pending(self):
|
|
|
|
print(f"[task][{self.name}] download queue already running")
|
|
|
|
return
|
|
|
|
|
|
|
|
manager.init(self)
|
2023-03-15 06:28:19 +00:00
|
|
|
downloader = VideoDownloader(task=self)
|
2023-03-02 07:17:03 +00:00
|
|
|
if from_queue:
|
|
|
|
downloader.add_pending()
|
2023-03-01 12:12:27 +00:00
|
|
|
downloader.run_queue()
|
2021-09-05 17:10:14 +00:00
|
|
|
|
|
|
|
|
2023-03-14 09:40:05 +00:00
|
|
|
@shared_task(name="extract_download", bind=True, base=BaseTask)
|
|
|
|
def extrac_dl(self, youtube_ids):
|
2021-09-21 09:25:22 +00:00
|
|
|
"""parse list passed and add to pending"""
|
2023-03-14 09:40:05 +00:00
|
|
|
pending_handler = PendingList(youtube_ids=youtube_ids, task=self)
|
2022-03-18 11:27:25 +00:00
|
|
|
pending_handler.parse_url_list()
|
|
|
|
pending_handler.add_to_pending()
|
2021-09-11 10:55:44 +00:00
|
|
|
|
|
|
|
|
2023-03-15 09:59:20 +00:00
|
|
|
@shared_task(bind=True, name="check_reindex", base=BaseTask)
|
2023-03-02 04:26:48 +00:00
|
|
|
def check_reindex(self, data=False, extract_videos=False):
|
2021-09-21 09:25:22 +00:00
|
|
|
"""run the reindex main command"""
|
2022-12-11 08:56:44 +00:00
|
|
|
if data:
|
2023-03-02 04:26:48 +00:00
|
|
|
# started from frontend through API
|
|
|
|
print(f"[task][{self.name}] reindex {data}")
|
2023-03-15 09:59:20 +00:00
|
|
|
self.send_progress("Add items to the reindex Queue.")
|
2022-12-11 08:56:44 +00:00
|
|
|
ReindexManual(extract_videos=extract_videos).extract_data(data)
|
|
|
|
|
2023-03-02 04:26:48 +00:00
|
|
|
manager = TaskManager()
|
|
|
|
if manager.is_pending(self):
|
|
|
|
print(f"[task][{self.name}] reindex queue is already running")
|
|
|
|
return
|
2022-12-12 10:33:57 +00:00
|
|
|
|
2023-03-02 04:26:48 +00:00
|
|
|
manager.init(self)
|
|
|
|
if not data:
|
|
|
|
# started from scheduler
|
|
|
|
print(f"[task][{self.name}] reindex outdated documents")
|
2023-03-15 09:59:20 +00:00
|
|
|
self.send_progress("Add outdated documents to the reindex Queue.")
|
2023-03-02 04:26:48 +00:00
|
|
|
ReindexOutdated().add_outdated()
|
2022-12-12 10:33:57 +00:00
|
|
|
|
2023-03-15 09:59:20 +00:00
|
|
|
Reindex(task=self).reindex_all()
|
2021-09-13 15:17:36 +00:00
|
|
|
|
|
|
|
|
2023-03-15 10:39:37 +00:00
|
|
|
@shared_task(bind=True, name="manual_import", base=BaseTask)
|
2023-03-02 04:26:48 +00:00
|
|
|
def run_manual_import(self):
|
2021-09-21 09:25:22 +00:00
|
|
|
"""called from settings page, to go through import folder"""
|
2023-03-02 04:26:48 +00:00
|
|
|
manager = TaskManager()
|
|
|
|
if manager.is_pending(self):
|
|
|
|
print(f"[task][{self.name}] manual import is already running")
|
|
|
|
return
|
2021-09-14 11:25:28 +00:00
|
|
|
|
2023-03-02 04:26:48 +00:00
|
|
|
manager.init(self)
|
2023-03-15 10:39:37 +00:00
|
|
|
ImportFolderScanner(task=self).scan()
|
2021-09-16 11:16:09 +00:00
|
|
|
|
2021-09-18 13:02:54 +00:00
|
|
|
|
2023-03-16 03:11:10 +00:00
|
|
|
@shared_task(bind=True, name="run_backup", base=BaseTask)
|
2023-03-02 04:26:48 +00:00
|
|
|
def run_backup(self, reason="auto"):
|
2021-09-21 09:25:22 +00:00
|
|
|
"""called from settings page, dump backup to zip file"""
|
2023-03-02 04:26:48 +00:00
|
|
|
manager = TaskManager()
|
|
|
|
if manager.is_pending(self):
|
|
|
|
print(f"[task][{self.name}] backup is already running")
|
|
|
|
return
|
2022-08-12 05:03:09 +00:00
|
|
|
|
2023-03-02 04:26:48 +00:00
|
|
|
manager.init(self)
|
2023-03-16 03:11:10 +00:00
|
|
|
ElasticBackup(reason=reason, task=self).backup_all_indexes()
|
2021-09-20 13:26:28 +00:00
|
|
|
|
|
|
|
|
2023-03-16 03:11:10 +00:00
|
|
|
@shared_task(bind=True, name="restore_backup", base=BaseTask)
|
2023-03-02 04:53:38 +00:00
|
|
|
def run_restore_backup(self, filename):
|
2021-09-21 09:25:22 +00:00
|
|
|
"""called from settings page, dump backup to zip file"""
|
2023-03-02 04:53:38 +00:00
|
|
|
manager = TaskManager()
|
|
|
|
if manager.is_pending(self):
|
|
|
|
print(f"[task][{self.name}] restore is already running")
|
|
|
|
return
|
|
|
|
|
|
|
|
manager.init(self)
|
2023-03-16 03:11:10 +00:00
|
|
|
self.send_progress(["Reset your Index"])
|
2022-11-27 08:41:59 +00:00
|
|
|
ElasitIndexWrap().reset()
|
2023-03-16 03:11:10 +00:00
|
|
|
ElasticBackup(task=self).restore(filename)
|
2021-09-21 09:25:22 +00:00
|
|
|
print("index restore finished")
|
2021-09-24 16:37:26 +00:00
|
|
|
|
|
|
|
|
2023-03-01 08:31:17 +00:00
|
|
|
@shared_task(name="kill_download")
|
2021-09-24 16:37:26 +00:00
|
|
|
def kill_dl(task_id):
|
|
|
|
"""kill download worker task by ID"""
|
2021-10-11 08:26:31 +00:00
|
|
|
if task_id:
|
|
|
|
app.control.revoke(task_id, terminate=True)
|
|
|
|
|
2021-09-28 03:33:00 +00:00
|
|
|
_ = RedisArchivist().del_message("dl_queue_id")
|
2022-12-10 07:51:17 +00:00
|
|
|
RedisQueue(queue_name="dl_queue").clear()
|
2021-09-24 16:37:26 +00:00
|
|
|
|
2023-02-02 03:46:59 +00:00
|
|
|
_ = clear_dl_cache(CONFIG)
|
2021-09-24 16:37:26 +00:00
|
|
|
|
|
|
|
# notify
|
|
|
|
mess_dict = {
|
2021-12-03 12:47:43 +00:00
|
|
|
"status": "message:download",
|
2021-09-24 16:37:26 +00:00
|
|
|
"level": "error",
|
2021-12-03 12:47:43 +00:00
|
|
|
"title": "Canceling download process",
|
|
|
|
"message": "Canceling download queue now.",
|
2021-09-24 16:37:26 +00:00
|
|
|
}
|
2022-06-16 03:37:46 +00:00
|
|
|
RedisArchivist().set_message("message:download", mess_dict, expire=True)
|
2021-10-08 07:56:07 +00:00
|
|
|
|
|
|
|
|
2023-03-16 06:18:26 +00:00
|
|
|
@shared_task(bind=True, name="rescan_filesystem", base=BaseTask)
|
2023-03-02 04:53:38 +00:00
|
|
|
def rescan_filesystem(self):
|
2021-10-08 08:10:44 +00:00
|
|
|
"""check the media folder for mismatches"""
|
2023-03-02 04:53:38 +00:00
|
|
|
manager = TaskManager()
|
|
|
|
if manager.is_pending(self):
|
|
|
|
print(f"[task][{self.name}] filesystem rescan already running")
|
|
|
|
return
|
|
|
|
|
|
|
|
manager.init(self)
|
2023-03-16 06:18:26 +00:00
|
|
|
Filesystem(task=self).process()
|
2023-03-16 10:12:50 +00:00
|
|
|
ThumbValidator(task=self).validate()
|
2021-10-30 08:00:10 +00:00
|
|
|
|
|
|
|
|
2023-03-16 10:12:50 +00:00
|
|
|
@shared_task(bind=True, name="thumbnail_check", base=BaseTask)
|
2023-03-02 04:53:38 +00:00
|
|
|
def thumbnail_check(self):
|
2021-12-02 08:54:29 +00:00
|
|
|
"""validate thumbnails"""
|
2023-03-02 04:53:38 +00:00
|
|
|
manager = TaskManager()
|
|
|
|
if manager.is_pending(self):
|
|
|
|
print(f"[task][{self.name}] thumbnail check is already running")
|
|
|
|
return
|
|
|
|
|
|
|
|
manager.init(self)
|
2023-03-16 10:12:50 +00:00
|
|
|
ThumbValidator(task=self).validate()
|
2021-12-02 08:54:29 +00:00
|
|
|
|
|
|
|
|
2023-03-16 11:13:37 +00:00
|
|
|
@shared_task(bind=True, name="resync_thumbs", base=BaseTask)
|
2023-03-02 04:53:38 +00:00
|
|
|
def re_sync_thumbs(self):
|
2021-11-01 09:20:21 +00:00
|
|
|
"""sync thumbnails to mediafiles"""
|
2023-03-02 04:53:38 +00:00
|
|
|
manager = TaskManager()
|
|
|
|
if manager.is_pending(self):
|
|
|
|
print(f"[task][{self.name}] thumb re-embed is already running")
|
|
|
|
return
|
|
|
|
|
|
|
|
manager.init(self)
|
2023-03-16 11:13:37 +00:00
|
|
|
ThumbFilesystem(task=self).embed()
|
2021-11-01 09:20:21 +00:00
|
|
|
|
|
|
|
|
2023-03-01 08:31:17 +00:00
|
|
|
@shared_task(name="subscribe_to")
|
2021-10-31 10:48:56 +00:00
|
|
|
def subscribe_to(url_str):
|
2021-10-30 08:00:10 +00:00
|
|
|
"""take a list of urls to subscribe to"""
|
2023-03-02 05:30:48 +00:00
|
|
|
SubscriptionHandler(url_str).subscribe()
|
2021-11-08 07:54:17 +00:00
|
|
|
|
|
|
|
|
2023-03-18 05:59:16 +00:00
|
|
|
@shared_task(bind=True, name="index_playlists", base=BaseTask)
|
|
|
|
def index_channel_playlists(self, channel_id):
|
2021-11-08 07:54:17 +00:00
|
|
|
"""add all playlists of channel to index"""
|
2023-03-18 05:59:16 +00:00
|
|
|
channel = YoutubeChannel(channel_id, task=self)
|
2022-03-21 10:59:21 +00:00
|
|
|
channel.index_channel_playlists()
|
2021-12-02 08:54:29 +00:00
|
|
|
|
|
|
|
|
2022-12-21 07:24:24 +00:00
|
|
|
@shared_task(name="version_check")
|
|
|
|
def version_check():
|
|
|
|
"""check for new updates"""
|
|
|
|
ReleaseVersion().check()
|
|
|
|
|
|
|
|
|
2023-02-02 06:06:00 +00:00
|
|
|
# start schedule here
|
2023-01-16 02:13:58 +00:00
|
|
|
app.conf.beat_schedule = ScheduleBuilder().build_schedule()
|