Support wider spectrum of ElasticSearch configurations
This commit is contained in:
parent
38b3815a33
commit
b2bb7ea28e
|
@ -59,6 +59,8 @@ Take a look at the example [docker-compose.yml](https://github.com/tubearchivist
|
||||||
| TA_PORT | Overwrite Nginx port | Optional |
|
| TA_PORT | Overwrite Nginx port | Optional |
|
||||||
| TA_UWSGI_PORT | Overwrite container internal uwsgi port | Optional |
|
| TA_UWSGI_PORT | Overwrite container internal uwsgi port | Optional |
|
||||||
| ES_URL | URL That ElasticSearch runs on | Optional |
|
| ES_URL | URL That ElasticSearch runs on | Optional |
|
||||||
|
| ES_VERIFY_SSL | Verify ElasticSearch SSL certificate, everything other than `false` defaults to `true` | Optional |
|
||||||
|
| ES_SNAPSHOT_DIR | Custom path where elastic search stores snapshots for master/data nodes | Optional |
|
||||||
| HOST_GID | Allow TA to own the video files instead of container user | Optional |
|
| HOST_GID | Allow TA to own the video files instead of container user | Optional |
|
||||||
| HOST_UID | Allow TA to own the video files instead of container user | Optional |
|
| HOST_UID | Allow TA to own the video files instead of container user | Optional |
|
||||||
| ELASTIC_USER | Change the default ElasticSearch user | Optional |
|
| ELASTIC_USER | Change the default ElasticSearch user | Optional |
|
||||||
|
|
|
@ -3,6 +3,7 @@ Functionality:
|
||||||
- check that all connections are working
|
- check that all connections are working
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
from os import environ
|
||||||
from time import sleep
|
from time import sleep
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
|
@ -132,7 +133,19 @@ class Command(BaseCommand):
|
||||||
"""check that path.repo var is set"""
|
"""check that path.repo var is set"""
|
||||||
self.stdout.write("[5] check ES path.repo env var")
|
self.stdout.write("[5] check ES path.repo env var")
|
||||||
response, _ = ElasticWrap("_nodes/_all/settings").get()
|
response, _ = ElasticWrap("_nodes/_all/settings").get()
|
||||||
|
snaphost_roles = [
|
||||||
|
"data",
|
||||||
|
"data_cold",
|
||||||
|
"data_content",
|
||||||
|
"data_frozen",
|
||||||
|
"data_hot",
|
||||||
|
"data_warm",
|
||||||
|
"master",
|
||||||
|
]
|
||||||
for node in response["nodes"].values():
|
for node in response["nodes"].values():
|
||||||
|
if not (set(node["roles"]) & set(snaphost_roles)):
|
||||||
|
continue
|
||||||
|
|
||||||
if node["settings"]["path"].get("repo"):
|
if node["settings"]["path"].get("repo"):
|
||||||
self.stdout.write(
|
self.stdout.write(
|
||||||
self.style.SUCCESS(" ✓ path.repo env var is set")
|
self.style.SUCCESS(" ✓ path.repo env var is set")
|
||||||
|
@ -142,7 +155,10 @@ class Command(BaseCommand):
|
||||||
message = (
|
message = (
|
||||||
" 🗙 path.repo env var not found. "
|
" 🗙 path.repo env var not found. "
|
||||||
+ "set the following env var to the ES container:\n"
|
+ "set the following env var to the ES container:\n"
|
||||||
+ " path.repo=/usr/share/elasticsearch/data/snapshot"
|
+ " path.repo="
|
||||||
|
+ environ.get(
|
||||||
|
"ES_SNAPSHOT_DIR", "/usr/share/elasticsearch/data/snapshot"
|
||||||
|
),
|
||||||
)
|
)
|
||||||
self.stdout.write(self.style.ERROR(f"{message}"))
|
self.stdout.write(self.style.ERROR(f"{message}"))
|
||||||
sleep(60)
|
sleep(60)
|
||||||
|
|
|
@ -9,6 +9,7 @@ import json
|
||||||
import os
|
import os
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
|
import urllib3
|
||||||
|
|
||||||
|
|
||||||
class ElasticWrap:
|
class ElasticWrap:
|
||||||
|
@ -19,19 +20,33 @@ class ElasticWrap:
|
||||||
ES_URL: str = str(os.environ.get("ES_URL"))
|
ES_URL: str = str(os.environ.get("ES_URL"))
|
||||||
ES_PASS: str = str(os.environ.get("ELASTIC_PASSWORD"))
|
ES_PASS: str = str(os.environ.get("ELASTIC_PASSWORD"))
|
||||||
ES_USER: str = str(os.environ.get("ELASTIC_USER") or "elastic")
|
ES_USER: str = str(os.environ.get("ELASTIC_USER") or "elastic")
|
||||||
|
ES_VERIFY_SSL: str = str(os.environ.get("ES_VERIFY_SSL") or "true")
|
||||||
|
|
||||||
def __init__(self, path):
|
def __init__(self, path):
|
||||||
self.url = f"{self.ES_URL}/{path}"
|
self.url = f"{self.ES_URL}/{path}"
|
||||||
self.auth = (self.ES_USER, self.ES_PASS)
|
self.auth = (self.ES_USER, self.ES_PASS)
|
||||||
|
self.verify_ssl = self.ES_VERIFY_SSL != "false"
|
||||||
|
|
||||||
|
if not self.verify_ssl:
|
||||||
|
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||||||
|
|
||||||
def get(self, data=False, timeout=10, print_error=True):
|
def get(self, data=False, timeout=10, print_error=True):
|
||||||
"""get data from es"""
|
"""get data from es"""
|
||||||
if data:
|
if data:
|
||||||
response = requests.get(
|
response = requests.get(
|
||||||
self.url, json=data, auth=self.auth, timeout=timeout
|
self.url,
|
||||||
|
json=data,
|
||||||
|
auth=self.auth,
|
||||||
|
timeout=timeout,
|
||||||
|
verify=self.verify_ssl,
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
response = requests.get(self.url, auth=self.auth, timeout=timeout)
|
response = requests.get(
|
||||||
|
self.url,
|
||||||
|
auth=self.auth,
|
||||||
|
timeout=timeout,
|
||||||
|
verify=self.verify_ssl,
|
||||||
|
)
|
||||||
if print_error and not response.ok:
|
if print_error and not response.ok:
|
||||||
print(response.text)
|
print(response.text)
|
||||||
|
|
||||||
|
@ -48,10 +63,19 @@ class ElasticWrap:
|
||||||
|
|
||||||
if data:
|
if data:
|
||||||
response = requests.post(
|
response = requests.post(
|
||||||
self.url, data=payload, headers=headers, auth=self.auth
|
self.url,
|
||||||
|
data=payload,
|
||||||
|
headers=headers,
|
||||||
|
auth=self.auth,
|
||||||
|
verify=self.verify_ssl,
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
response = requests.post(self.url, headers=headers, auth=self.auth)
|
response = requests.post(
|
||||||
|
self.url,
|
||||||
|
headers=headers,
|
||||||
|
auth=self.auth,
|
||||||
|
verify=self.verify_ssl,
|
||||||
|
)
|
||||||
|
|
||||||
if not response.ok:
|
if not response.ok:
|
||||||
print(response.text)
|
print(response.text)
|
||||||
|
@ -62,7 +86,9 @@ class ElasticWrap:
|
||||||
"""put data to es"""
|
"""put data to es"""
|
||||||
if refresh:
|
if refresh:
|
||||||
self.url = f"{self.url}/?refresh=true"
|
self.url = f"{self.url}/?refresh=true"
|
||||||
response = requests.put(f"{self.url}", json=data, auth=self.auth)
|
response = requests.put(
|
||||||
|
f"{self.url}", json=data, auth=self.auth, verify=self.verify_ssl
|
||||||
|
)
|
||||||
if not response.ok:
|
if not response.ok:
|
||||||
print(response.text)
|
print(response.text)
|
||||||
print(data)
|
print(data)
|
||||||
|
@ -75,9 +101,13 @@ class ElasticWrap:
|
||||||
if refresh:
|
if refresh:
|
||||||
self.url = f"{self.url}/?refresh=true"
|
self.url = f"{self.url}/?refresh=true"
|
||||||
if data:
|
if data:
|
||||||
response = requests.delete(self.url, json=data, auth=self.auth)
|
response = requests.delete(
|
||||||
|
self.url, json=data, auth=self.auth, verify=self.verify_ssl
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
response = requests.delete(self.url, auth=self.auth)
|
response = requests.delete(
|
||||||
|
self.url, auth=self.auth, verify=self.verify_ssl
|
||||||
|
)
|
||||||
|
|
||||||
if not response.ok:
|
if not response.ok:
|
||||||
print(response.text)
|
print(response.text)
|
||||||
|
|
|
@ -19,7 +19,9 @@ class ElasticSnapshot:
|
||||||
REPO_SETTINGS = {
|
REPO_SETTINGS = {
|
||||||
"compress": "true",
|
"compress": "true",
|
||||||
"chunk_size": "1g",
|
"chunk_size": "1g",
|
||||||
"location": "/usr/share/elasticsearch/data/snapshot",
|
"location": environ.get(
|
||||||
|
"ES_SNAPSHOT_DIR", "/usr/share/elasticsearch/data/snapshot"
|
||||||
|
),
|
||||||
}
|
}
|
||||||
POLICY = "ta_daily"
|
POLICY = "ta_daily"
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue