tubearchivist
/
tubearchivist
mirror of https://github.com/tubearchivist/tubearchivist.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

use ElasticWrap in FilesystemScanner

This commit is contained in:
simon 2022-03-23 15:56:53 +07:00
parent fda520ad44
commit 912c19f6cf
No known key found for this signature in database
GPG Key ID: 2C15AA5E89985DD4
1 changed files with 5 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
tubearchivist/home/src/index/filesystem.py
View File

@ -11,9 +11,9 @@ import re
import shutil import shutil
import subprocess import subprocess
import requests
from home.src.download.queue import PendingList from home.src.download.queue import PendingList
from home.src.download.yt_dlp_handler import VideoDownloader from home.src.download.yt_dlp_handler import VideoDownloader
from home.src.es.connect import ElasticWrap
from home.src.index.reindex import Reindex from home.src.index.reindex import Reindex
from home.src.index.video import index_new_video from home.src.index.video import index_new_video
from home.src.ta.config import AppConfig from home.src.ta.config import AppConfig
@ -25,8 +25,6 @@ class FilesystemScanner:
"""handle scanning and fixing from filesystem""" """handle scanning and fixing from filesystem"""
CONFIG = AppConfig().config CONFIG = AppConfig().config
ES_URL = CONFIG["application"]["es_url"]
ES_AUTH = CONFIG["application"]["es_auth"]
VIDEOS = CONFIG["application"]["videos"] VIDEOS = CONFIG["application"]["videos"]
def __init__(self): def __init__(self):
@ -147,25 +145,16 @@ class FilesystemScanner:
bulk_list.append(json.dumps(source)) bulk_list.append(json.dumps(source))
# add last newline # add last newline
bulk_list.append("\n") bulk_list.append("\n")
query_str = "\n".join(bulk_list) data = "\n".join(bulk_list)
# make the call _, _ = ElasticWrap("_bulk").post(data=data, ndjson=True)
headers = {"Content-type": "application/x-ndjson"}
url = self.ES_URL + "/_bulk"
request = requests.post(
url, data=query_str, headers=headers, auth=self.ES_AUTH
)
if not request.ok:
print(request.text)
def delete_from_index(self): def delete_from_index(self):
"""find indexed but deleted mediafile""" """find indexed but deleted mediafile"""
for indexed in self.to_delete: for indexed in self.to_delete:
youtube_id = indexed[0] youtube_id = indexed[0]
print(f"deleting {youtube_id} from index") print(f"deleting {youtube_id} from index")
url = self.ES_URL + "/ta_video/_doc/" + youtube_id path = f"ta_video/_doc/{youtube_id}"
request = requests.delete(url, auth=self.ES_AUTH) _, _ = ElasticWrap(path).delete()
if not request.ok:
print(request.text)
class ManualImport: class ManualImport: