backup es db to zip file

backup es db to zip file
This commit is contained in:
simon 2021-09-16 17:34:20 +07:00
parent 9234164ff6
commit e53cfc5a32
2 changed files with 44 additions and 9 deletions

View File

@ -22,8 +22,8 @@ def sync_redis_state():
def make_folders():
""" make needed folders here to avoid letting docker messing it up """
folders = ['download', 'channels', 'videos', 'import']
""" make needed cache folders here so docker doesn't mess it up """
folders = ['download', 'channels', 'videos', 'import', 'backup']
config = AppConfig().config
cache_dir = config['application']['cache_dir']
for folder in folders:

View File

@ -8,6 +8,7 @@ Functionality:
import json
import os
import zipfile
from datetime import datetime
@ -375,6 +376,7 @@ class ElasticBackup:
self.config = AppConfig().config
self.index_config = index_config
self.timestamp = datetime.now().strftime('%Y%m%d')
self.backup_files = []
def get_all_documents(self, index_name):
""" export all documents of a single index """
@ -433,14 +435,44 @@ class ElasticBackup:
return file_content
def write_json_file(self, file_content, index_name):
""" write json file to disk """
def write_es_json(self, file_content, index_name):
""" write nd json file for es _bulk API to disk """
cache_dir = self.config['application']['cache_dir']
file_name = f'ta_{index_name}-{self.timestamp}.json'
file_path = os.path.join(cache_dir, file_name)
file_name = f'es_{index_name}-{self.timestamp}.json'
file_path = os.path.join(cache_dir, 'backup', file_name)
with open(file_path, 'w', encoding='utf-8') as f:
f.write(file_content)
self.backup_files.append(file_path)
def write_ta_json(self, all_results, index_name):
""" write generic json file to disk """
cache_dir = self.config['application']['cache_dir']
file_name = f'ta_{index_name}-{self.timestamp}.json'
file_path = os.path.join(cache_dir, 'backup', file_name)
to_write = [i['_source'] for i in all_results]
file_content = json.dumps(to_write)
with open(file_path, 'w', encoding='utf-8') as f:
f.write(file_content)
self.backup_files.append(file_path)
def zip_it(self):
""" pack it up into single zip file """
cache_dir = self.config['application']['cache_dir']
file_name = f'ta_backup-{self.timestamp}.zip'
backup_file = os.path.join(cache_dir, 'backup', file_name)
with zipfile.ZipFile(
backup_file, 'w', compression=zipfile.ZIP_DEFLATED
) as zip_f:
for backup_file in self.backup_files:
zip_f.write(backup_file)
# cleanup
for backup_file in self.backup_files:
os.remove(backup_file)
def post_bulk_restore(self, file_name):
""" send bulk to es """
cache_dir = self.config['application']['cache_dir']
@ -475,7 +507,10 @@ def backup_all_indexes():
index_name = index['index_name']
all_results = backup_handler.get_all_documents(index_name)
file_content = backup_handler.build_bulk(all_results)
backup_handler.write_json_file(file_content, index_name)
backup_handler.write_es_json(file_content, index_name)
backup_handler.write_ta_json(all_results, index_name)
backup_handler.zip_it()
def restore_from_backup():