tubearchivist/tubearchivist-frontend
1
0
Fork 0
mirror of https://github.com/tubearchivist/tubearchivist-frontend.git synced 2024-11-04 19:30:13 +00:00
Code Issues Projects Releases Wiki Activity

backup es db to zip file

Browse Source 
backup es db to zip file
This commit is contained in:
simon 2021-09-16 17:34:20 +07:00
parent 9234164ff6
commit e53cfc5a32
2 changed files with 44 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
tubearchivist/home/__init__.py
View File

@ -22,8 +22,8 @@ def sync_redis_state():
def make_folders(): def make_folders():
""" make needed folders here to avoid letting docker messing it up """ """ make needed cache folders here so docker doesn't mess it up """
folders = ['download', 'channels', 'videos', 'import'] folders = ['download', 'channels', 'videos', 'import', 'backup']
config = AppConfig().config config = AppConfig().config
cache_dir = config['application']['cache_dir'] cache_dir = config['application']['cache_dir']
for folder in folders: for folder in folders:

45
tubearchivist/home/src/index_management.py
View File

@ -8,6 +8,7 @@ Functionality:
import json import json
import os import os
import zipfile
from datetime import datetime from datetime import datetime
@ -375,6 +376,7 @@ class ElasticBackup:
self.config = AppConfig().config self.config = AppConfig().config
self.index_config = index_config self.index_config = index_config
self.timestamp = datetime.now().strftime('%Y%m%d') self.timestamp = datetime.now().strftime('%Y%m%d')
self.backup_files = []
def get_all_documents(self, index_name): def get_all_documents(self, index_name):
""" export all documents of a single index """ """ export all documents of a single index """
@ -433,14 +435,44 @@ class ElasticBackup:
return file_content return file_content
def write_json_file(self, file_content, index_name): def write_es_json(self, file_content, index_name):
""" write json file to disk """ """ write nd json file for es _bulk API to disk """
cache_dir = self.config['application']['cache_dir'] cache_dir = self.config['application']['cache_dir']
file_name = f'ta_{index_name}-{self.timestamp}.json' file_name = f'es_{index_name}-{self.timestamp}.json'
file_path = os.path.join(cache_dir, file_name) file_path = os.path.join(cache_dir, 'backup', file_name)
with open(file_path, 'w', encoding='utf-8') as f: with open(file_path, 'w', encoding='utf-8') as f:
f.write(file_content) f.write(file_content)
self.backup_files.append(file_path)
def write_ta_json(self, all_results, index_name):
""" write generic json file to disk """
cache_dir = self.config['application']['cache_dir']
file_name = f'ta_{index_name}-{self.timestamp}.json'
file_path = os.path.join(cache_dir, 'backup', file_name)
to_write = [i['_source'] for i in all_results]
file_content = json.dumps(to_write)
with open(file_path, 'w', encoding='utf-8') as f:
f.write(file_content)
self.backup_files.append(file_path)
def zip_it(self):
""" pack it up into single zip file """
cache_dir = self.config['application']['cache_dir']
file_name = f'ta_backup-{self.timestamp}.zip'
backup_file = os.path.join(cache_dir, 'backup', file_name)
with zipfile.ZipFile(
backup_file, 'w', compression=zipfile.ZIP_DEFLATED
) as zip_f:
for backup_file in self.backup_files:
zip_f.write(backup_file)
# cleanup
for backup_file in self.backup_files:
os.remove(backup_file)
def post_bulk_restore(self, file_name): def post_bulk_restore(self, file_name):
""" send bulk to es """ """ send bulk to es """
cache_dir = self.config['application']['cache_dir'] cache_dir = self.config['application']['cache_dir']
@ -475,7 +507,10 @@ def backup_all_indexes():
index_name = index['index_name'] index_name = index['index_name']
all_results = backup_handler.get_all_documents(index_name) all_results = backup_handler.get_all_documents(index_name)
file_content = backup_handler.build_bulk(all_results) file_content = backup_handler.build_bulk(all_results)
backup_handler.write_json_file(file_content, index_name) backup_handler.write_es_json(file_content, index_name)
backup_handler.write_ta_json(all_results, index_name)
backup_handler.zip_it()
def restore_from_backup(): def restore_from_backup():