add recent videos to reindex queue task
This commit is contained in:
parent
ff89432d86
commit
d2649c29c3
|
@ -49,6 +49,7 @@ class ReindexBase:
|
|||
}
|
||||
|
||||
MULTIPLY = 1.2
|
||||
DAYS3 = 60 * 60 * 24 * 3
|
||||
|
||||
def __init__(self):
|
||||
self.config = AppConfig().config
|
||||
|
@ -62,13 +63,34 @@ class ReindexBase:
|
|||
RedisQueue(queue_name=reindex_config["queue_name"]).add_list(all_ids)
|
||||
|
||||
|
||||
class ReindexOutdated(ReindexBase):
|
||||
"""add outdated documents to reindex queue"""
|
||||
class ReindexPopulate(ReindexBase):
|
||||
"""add outdated and recent documents to reindex queue"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.interval = self.config["scheduler"]["check_reindex_days"]
|
||||
|
||||
def add_recent(self):
|
||||
"""add recent videos to refresh"""
|
||||
gte = datetime.fromtimestamp(self.now - self.DAYS3).date().isoformat()
|
||||
must_list = [
|
||||
{"term": {"active": {"value": True}}},
|
||||
{"range": {"published": {"gte": gte}}},
|
||||
]
|
||||
data = {
|
||||
"size": 10000,
|
||||
"query": {"bool": {"must": must_list}},
|
||||
"sort": [{"published": {"order": "desc"}}],
|
||||
}
|
||||
response, _ = ElasticWrap("ta_video/_search").get(data=data)
|
||||
hits = response["hits"]["hits"]
|
||||
if not hits:
|
||||
return
|
||||
|
||||
all_ids = [i["_source"]["youtube_id"] for i in hits]
|
||||
reindex_config = self.REINDEX_CONFIG.get("video")
|
||||
self.populate(all_ids, reindex_config)
|
||||
|
||||
def add_outdated(self):
|
||||
"""add outdated documents"""
|
||||
for reindex_config in self.REINDEX_CONFIG.values():
|
||||
|
|
|
@ -21,7 +21,7 @@ from home.src.es.index_setup import ElasitIndexWrap
|
|||
from home.src.index.channel import YoutubeChannel
|
||||
from home.src.index.filesystem import Filesystem
|
||||
from home.src.index.manual import ImportFolderScanner
|
||||
from home.src.index.reindex import Reindex, ReindexManual, ReindexOutdated
|
||||
from home.src.index.reindex import Reindex, ReindexManual, ReindexPopulate
|
||||
from home.src.ta.config import AppConfig, ReleaseVersion, ScheduleBuilder
|
||||
from home.src.ta.ta_redis import RedisArchivist
|
||||
from home.src.ta.task_manager import TaskManager
|
||||
|
@ -220,9 +220,12 @@ def check_reindex(self, data=False, extract_videos=False):
|
|||
manager.init(self)
|
||||
if not data:
|
||||
# started from scheduler
|
||||
populate = ReindexPopulate()
|
||||
print(f"[task][{self.name}] reindex outdated documents")
|
||||
self.send_progress("Add recent documents to the reindex Queue.")
|
||||
populate.add_recent()
|
||||
self.send_progress("Add outdated documents to the reindex Queue.")
|
||||
ReindexOutdated().add_outdated()
|
||||
populate.add_outdated()
|
||||
|
||||
Reindex(task=self).reindex_all()
|
||||
|
||||
|
|
Loading…
Reference in New Issue