tubearchivist
/
tubearchivist-frontend
mirror of https://github.com/tubearchivist/tubearchivist-frontend.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

use config check_reindex_days for refresh_interval

This commit is contained in:
simon 2021-12-13 18:13:29 +07:00
parent f660b05a20
commit 5cb6076a1c
1 changed files with 9 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
tubearchivist/home/src/reindex.py
View File

@ -41,7 +41,7 @@ class Reindex:
self.sleep_interval = config["downloads"]["sleep_interval"] self.sleep_interval = config["downloads"]["sleep_interval"]
self.es_url = config["application"]["es_url"] self.es_url = config["application"]["es_url"]
self.es_auth = config["application"]["es_auth"] self.es_auth = config["application"]["es_auth"]
self.refresh_interval = 90 self.refresh_interval = config["scheduler"]["check_reindex_days"]
# scan # scan
self.all_youtube_ids = False self.all_youtube_ids = False
self.all_channel_ids = False self.all_channel_ids = False
@ -57,23 +57,24 @@ class Reindex:
"ta_channel", self.es_url, self.es_auth, "channel_active" "ta_channel", self.es_url, self.es_auth, "channel_active"
) )
channel_daily = ceil(total_channels / self.refresh_interval * 1.2) channel_daily = ceil(total_channels / self.refresh_interval * 1.2)
playlist_daily = get_total_hits( total_playlists = get_total_hits(
"ta_playlist", self.es_url, self.es_auth, "playlist_active" "ta_playlist", self.es_url, self.es_auth, "playlist_active"
) )
playlist_daily = ceil(total_playlists / self.refresh_interval * 1.2)
return (video_daily, channel_daily, playlist_daily) return (video_daily, channel_daily, playlist_daily)
def get_outdated_vids(self, size): def get_outdated_vids(self, size):
"""get daily videos to refresh""" """get daily videos to refresh"""
headers = {"Content-type": "application/json"} headers = {"Content-type": "application/json"}
now = int(datetime.now().strftime("%s")) now = int(datetime.now().strftime("%s"))
now_3m = now - 3 * 30 * 24 * 60 * 60 now_lte = now - self.refresh_interval * 24 * 60 * 60
data = { data = {
"size": size, "size": size,
"query": { "query": {
"bool": { "bool": {
"must": [ "must": [
{"match": {"active": True}}, {"match": {"active": True}},
{"range": {"vid_last_refresh": {"lte": now_3m}}}, {"range": {"vid_last_refresh": {"lte": now_lte}}},
] ]
} }
}, },
@ -95,14 +96,14 @@ class Reindex:
"""get daily channels to refresh""" """get daily channels to refresh"""
headers = {"Content-type": "application/json"} headers = {"Content-type": "application/json"}
now = int(datetime.now().strftime("%s")) now = int(datetime.now().strftime("%s"))
now_3m = now - 3 * 30 * 24 * 60 * 60 now_lte = now - self.refresh_interval * 24 * 60 * 60
data = { data = {
"size": size, "size": size,
"query": { "query": {
"bool": { "bool": {
"must": [ "must": [
{"match": {"channel_active": True}}, {"match": {"channel_active": True}},
{"range": {"channel_last_refresh": {"lte": now_3m}}}, {"range": {"channel_last_refresh": {"lte": now_lte}}},
] ]
} }
}, },
@ -124,14 +125,14 @@ class Reindex:
"""get daily outdated playlists to refresh""" """get daily outdated playlists to refresh"""
headers = {"Content-type": "application/json"} headers = {"Content-type": "application/json"}
now = int(datetime.now().strftime("%s")) now = int(datetime.now().strftime("%s"))
now_3m = now - 3 * 30 * 24 * 60 * 60 now_lte = now - self.refresh_interval * 24 * 60 * 60
data = { data = {
"size": size, "size": size,
"query": { "query": {
"bool": { "bool": {
"must": [ "must": [
{"match": {"playlist_active": True}}, {"match": {"playlist_active": True}},
{"range": {"playlist_last_refresh": {"lte": now_3m}}}, {"range": {"playlist_last_refresh": {"lte": now_lte}}},
] ]
} }
}, },