tubearchivist/tubearchivist-frontend
1
0
Fork 0
mirror of https://github.com/tubearchivist/tubearchivist-frontend.git synced 2024-11-22 11:50:14 +00:00
Code Issues Projects Releases Wiki Activity

restoring indexes from zip archive

Browse Source
This commit is contained in:
simon 2021-09-18 17:10:16 +07:00
parent 82e4ee0350
commit eafda731c9
1 changed files with 37 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
tubearchivist/home/src/index_management.py
View File

@ -391,7 +391,7 @@ class ElasticBackup:
data = { data = {
"query": {"match_all": {}}, "query": {"match_all": {}},
"size": 100, "pit": {"id": pit_id, "keep_alive": "1m"}, "size": 100, "pit": {"id": pit_id, "keep_alive": "1m"},
"sort": [{"youtube_id": {"order": "asc"}}] "sort": [{"_id": {"order": "asc"}}]
} }
query_str = json.dumps(data) query_str = json.dumps(data)
url = es_url + '/_search' url = es_url + '/_search'
@ -461,13 +461,14 @@ class ElasticBackup:
""" pack it up into single zip file """ """ pack it up into single zip file """
cache_dir = self.config['application']['cache_dir'] cache_dir = self.config['application']['cache_dir']
file_name = f'ta_backup-{self.timestamp}.zip' file_name = f'ta_backup-{self.timestamp}.zip'
backup_file = os.path.join(cache_dir, 'backup', file_name) backup_folder = os.path.join(cache_dir, 'backup')
backup_file = os.path.join(backup_folder, file_name)
with zipfile.ZipFile( with zipfile.ZipFile(
backup_file, 'w', compression=zipfile.ZIP_DEFLATED backup_file, 'w', compression=zipfile.ZIP_DEFLATED
) as zip_f: ) as zip_f:
for backup_file in self.backup_files: for backup_file in self.backup_files:
zip_f.write(backup_file) zip_f.write(backup_file, os.path.basename(backup_file))
# cleanup # cleanup
for backup_file in self.backup_files: for backup_file in self.backup_files:
@ -488,15 +489,41 @@ class ElasticBackup:
if not request.ok: if not request.ok:
print(request.text) print(request.text)
def restore_from_file(self): def unpack_zip_backup(self):
""" restore all available backup files """ """ extract backup zip and return filelist """
cache_dir = self.config['application']['cache_dir'] cache_dir = self.config['application']['cache_dir']
backup_dir = os.path.join(cache_dir, 'backup')
all_available_backups = [ all_available_backups = [
i for i in os.listdir(cache_dir) if i for i in os.listdir(backup_dir) if
i.startswith('ta_') and i.endswith('.json') i.startswith('ta_') and i.endswith('.zip')
] ]
for file_name in all_available_backups: all_available_backups.sort()
newest_backup = all_available_backups[-1]
file_path = os.path.join(backup_dir, newest_backup)
with zipfile.ZipFile(file_path, 'r') as z:
zip_content = z.namelist()
z.extractall(backup_dir)
return zip_content
def restore_json_files(self, zip_content):
""" go through the unpacked files and restore """
cache_dir = self.config['application']['cache_dir']
backup_dir = os.path.join(cache_dir, 'backup')
for json_f in zip_content:
file_name = os.path.join(backup_dir, json_f)
if not json_f.startswith('es_') or not json_f.endswith('.json'):
os.remove(file_name)
continue
print('restoring: ' + json_f)
self.post_bulk_restore(file_name) self.post_bulk_restore(file_name)
os.remove(file_name)
def backup_all_indexes(): def backup_all_indexes():
@ -519,7 +546,8 @@ def restore_from_backup():
index_check(force_restore=True) index_check(force_restore=True)
# recreate # recreate
backup_handler = ElasticBackup(INDEX_CONFIG) backup_handler = ElasticBackup(INDEX_CONFIG)
backup_handler.restore_from_file() zip_content = backup_handler.unpack_zip_backup()
backup_handler.restore_json_files(zip_content)
def index_check(force_restore=False): def index_check(force_restore=False):