Support wider spectrum of ElasticSearch configurations

README.md

@@ -59,6 +59,8 @@ Take a look at the example [docker-compose.yml](https://github.com/tubearchivist
 | TA_PORT | Overwrite Nginx port | Optional |
 | TA_UWSGI_PORT | Overwrite container internal uwsgi port | Optional |
 | ES_URL | URL That ElasticSearch runs on | Optional |
+| ES_VERIFY_SSL | Verify ElasticSearch SSL certificate, everything other than `false` defaults to `true` | Optional |
+| ES_SNAPSHOT_DIR | Custom path where elastic search stores snapshots for master/data nodes | Optional |
 | HOST_GID | Allow TA to own the video files instead of container user | Optional |
 | HOST_UID | Allow TA to own the video files instead of container user | Optional |
 | ELASTIC_USER | Change the default ElasticSearch user | Optional |

tubearchivist/config/management/commands/ta_connection.py

@@ -3,6 +3,7 @@ Functionality:
 - check that all connections are working
 """
 
+from os import environ
 from time import sleep
 
 import requests
@@ -132,7 +133,19 @@ class Command(BaseCommand):
         """check that path.repo var is set"""
         self.stdout.write("[5] check ES path.repo env var")
         response, _ = ElasticWrap("_nodes/_all/settings").get()
+        snaphost_roles = [
+            "data",
+            "data_cold",
+            "data_content",
+            "data_frozen",
+            "data_hot",
+            "data_warm",
+            "master",
+        ]
         for node in response["nodes"].values():
+            if not (set(node["roles"]) & set(snaphost_roles)):
+                continue
+
             if node["settings"]["path"].get("repo"):
                 self.stdout.write(
                     self.style.SUCCESS("    ✓ path.repo env var is set")
@@ -142,7 +155,10 @@ class Command(BaseCommand):
             message = (
                 "    🗙 path.repo env var not found. "
                 + "set the following env var to the ES container:\n"
-                + "    path.repo=/usr/share/elasticsearch/data/snapshot"
+                + "    path.repo="
+                + environ.get(
+                    "ES_SNAPSHOT_DIR", "/usr/share/elasticsearch/data/snapshot"
+                ),
             )
             self.stdout.write(self.style.ERROR(f"{message}"))
             sleep(60)

tubearchivist/home/src/es/connect.py

@@ -9,6 +9,7 @@ import json
 import os
 
 import requests
+import urllib3
 
 
 class ElasticWrap:
@@ -19,19 +20,33 @@ class ElasticWrap:
     ES_URL: str = str(os.environ.get("ES_URL"))
     ES_PASS: str = str(os.environ.get("ELASTIC_PASSWORD"))
     ES_USER: str = str(os.environ.get("ELASTIC_USER") or "elastic")
+    ES_VERIFY_SSL: str = str(os.environ.get("ES_VERIFY_SSL") or "true")
 
     def __init__(self, path):
         self.url = f"{self.ES_URL}/{path}"
         self.auth = (self.ES_USER, self.ES_PASS)
+        self.verify_ssl = self.ES_VERIFY_SSL != "false"
+
+        if not self.verify_ssl:
+            urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
     def get(self, data=False, timeout=10, print_error=True):
         """get data from es"""
         if data:
             response = requests.get(
-                self.url, json=data, auth=self.auth, timeout=timeout
+                self.url,
+                json=data,
+                auth=self.auth,
+                timeout=timeout,
+                verify=self.verify_ssl,
             )
         else:
-            response = requests.get(self.url, auth=self.auth, timeout=timeout)
+            response = requests.get(
+                self.url,
+                auth=self.auth,
+                timeout=timeout,
+                verify=self.verify_ssl,
+            )
         if print_error and not response.ok:
             print(response.text)
 
@@ -48,10 +63,19 @@ class ElasticWrap:
 
         if data:
             response = requests.post(
-                self.url, data=payload, headers=headers, auth=self.auth
+                self.url,
+                data=payload,
+                headers=headers,
+                auth=self.auth,
+                verify=self.verify_ssl,
             )
         else:
-            response = requests.post(self.url, headers=headers, auth=self.auth)
+            response = requests.post(
+                self.url,
+                headers=headers,
+                auth=self.auth,
+                verify=self.verify_ssl,
+            )
 
         if not response.ok:
             print(response.text)
@@ -62,7 +86,9 @@ class ElasticWrap:
         """put data to es"""
         if refresh:
             self.url = f"{self.url}/?refresh=true"
-        response = requests.put(f"{self.url}", json=data, auth=self.auth)
+        response = requests.put(
+            f"{self.url}", json=data, auth=self.auth, verify=self.verify_ssl
+        )
         if not response.ok:
             print(response.text)
             print(data)
@@ -75,9 +101,13 @@ class ElasticWrap:
         if refresh:
             self.url = f"{self.url}/?refresh=true"
         if data:
-            response = requests.delete(self.url, json=data, auth=self.auth)
+            response = requests.delete(
+                self.url, json=data, auth=self.auth, verify=self.verify_ssl
+            )
         else:
-            response = requests.delete(self.url, auth=self.auth)
+            response = requests.delete(
+                self.url, auth=self.auth, verify=self.verify_ssl
+            )
 
         if not response.ok:
             print(response.text)

tubearchivist/home/src/es/snapshot.py

@@ -19,7 +19,9 @@ class ElasticSnapshot:
     REPO_SETTINGS = {
         "compress": "true",
         "chunk_size": "1g",
-        "location": "/usr/share/elasticsearch/data/snapshot",
+        "location": environ.get(
+            "ES_SNAPSHOT_DIR", "/usr/share/elasticsearch/data/snapshot"
+        ),
     }
     POLICY = "ta_daily"