add envcheck and basic connection check startup command

docker_assets/run.sh

@@ -1,71 +1,30 @@
 #!/bin/bash
 # startup script inside the container for tubearchivist
 
-if [[ -z "$ELASTIC_USER" ]]; then
-    export ELASTIC_USER=elastic
-fi
 
-cachedir=/cache
-[[ -d $cachedir ]] || cachedir=.
-lockfile=${cachedir}/initsu.lock
-
-required="Missing required environment variable"
-[[ -f $lockfile ]] || : "${TA_USERNAME:?$required}"
-: "${TA_PASSWORD:?$required}"
-: "${ELASTIC_PASSWORD:?$required}"
-: "${TA_HOST:?$required}"
-
-# ugly nginx and uwsgi port overwrite with env vars
-if [[ -n "$TA_PORT" ]]; then
-    sed -i "s/8000/$TA_PORT/g" /etc/nginx/sites-available/default
-fi
-
-if [[ -n "$TA_UWSGI_PORT" ]]; then
-    sed -i "s/8080/$TA_UWSGI_PORT/g" /etc/nginx/sites-available/default
-    sed -i "s/8080/$TA_UWSGI_PORT/g" /app/uwsgi.ini
-fi
-
-# disable auth on static files for cast support
-if [[ -n "$ENABLE_CAST" ]]; then
-    sed -i "/auth_request/d" /etc/nginx/sites-available/default
-fi
+# check environment
+python manage.py ta_envcheck
+python manage.py ta_connection
 
 # wait for elasticsearch
-counter=0
-until curl -u "$ELASTIC_USER":"$ELASTIC_PASSWORD" "$ES_URL" -fs; do
-    echo "waiting for elastic search to start"
-    counter=$((counter+1))
-    if [[ $counter -eq 12 ]]; then
-        # fail after 2 min
-        echo "failed to connect to elastic search, exiting..."
-        curl -v -u "$ELASTIC_USER":"$ELASTIC_PASSWORD" "$ES_URL"?pretty
-        exit 1
-    fi
-    sleep 10
-done
+# counter=0
+# until curl -u "$ELASTIC_USER":"$ELASTIC_PASSWORD" "$ES_URL" -fs; do
+#     echo "waiting for elastic search to start"
+#     counter=$((counter+1))
+#     if [[ $counter -eq 12 ]]; then
+#         # fail after 2 min
+#         echo "failed to connect to elastic search, exiting..."
+#         curl -v -u "$ELASTIC_USER":"$ELASTIC_PASSWORD" "$ES_URL"?pretty
+#         exit 1
+#     fi
+#     sleep 10
+# done
 
 # start python application
 python manage.py makemigrations
 python manage.py migrate
+# python manage.py collectstatic --noinput -c
 
-if [[ -f $lockfile ]]; then
-    echo -e "\e[33;1m[WARNING]\e[0m This is not the first run! Skipping" \
-        "superuser creation.\nTo force it, remove $lockfile"
-else
-    export DJANGO_SUPERUSER_PASSWORD=$TA_PASSWORD
-    output="$(python manage.py createsuperuser --noinput --name "$TA_USERNAME" 2>&1)"
-
-    case "$output" in
-        *"Superuser created successfully"*)
-            echo "$output" && touch $lockfile ;;
-        *"That name is already taken."*)
-            echo "Superuser already exists. Creation will be skipped on next start."
-            touch $lockfile ;;
-        *) echo "$output" && exit 1
-    esac
-fi
-
-python manage.py collectstatic --noinput -c
 nginx &
 celery -A home.tasks worker --loglevel=INFO &
 celery -A home beat --loglevel=INFO \

tubearchivist/config/management/commands/ta_connection.py

@@ -0,0 +1,57 @@
+"""
+Functionality:
+- check that all connections are working
+"""
+
+import sys
+from time import sleep
+
+import requests
+from django.core.management.base import BaseCommand, CommandError
+from home.src.es.connect import ElasticWrap
+
+
+TOPIC = """
+
+#######################
+#  Connection check   #
+#######################
+
+"""
+
+class Command(BaseCommand):
+    """command framework"""
+
+    TIMEOUT = 120
+
+    # pylint: disable=no-member
+    help = "Check connections"
+
+    def handle(self, *args, **options):
+        """run all commands"""
+        self.stdout.write(TOPIC)
+        self._es_connection_check()
+
+    def _es_connection_check(self):
+        """wait for elasticsearch connection"""
+        self.stdout.write("[1] connect to Elastic Search")
+        sys.stdout.write("    .")
+        for _ in range(self.TIMEOUT // 5):
+            sleep(2)
+            sys.stdout.write(".")
+            try:
+                response, status_code = ElasticWrap("/").get(
+                    timeout=1, print_error=False
+                )
+            except requests.exceptions.ConnectionError:
+                pass
+
+            if status_code == 200:
+                self.stdout.write("\n    ✓ ES connection established")
+                return
+
+        message = "    🗙 ES connection failed"
+        self.stdout.write(self.style.ERROR(f"\n{message}"))
+        self.stdout.write(f"    error message: {response | None}")
+        self.stdout.write(f"    status code: {status_code | None}")
+        raise CommandError(message)

tubearchivist/config/management/commands/ta_envcheck.py

@@ -0,0 +1,195 @@
+"""
+Functionality:
+- Check environment at startup
+- Process config file overwrites from env var
+- Stop startup on error
+- python management.py ta_envcheck
+"""
+
+import os
+import re
+
+from django.core.management.base import BaseCommand, CommandError
+from home.models import Account
+
+LOGO = """
+
+                         ....  .....
+                  ...'',;:cc,. .;::;;,'...
+               ..,;:cccllclc,  .:ccllllcc;,..
+            ..,:cllcc:;,'.',.  ....'',;ccllc:,..
+          ..;cllc:,'..                ...,:cccc:'.
+         .;cccc;..                        ..,:ccc:'.
+       .ckkkOkxollllllllllllc.      .,:::;.  .,cclc;
+      .:0MMMMMMMMMMMMMMMMMMMX:     .cNMMMWx.   .;clc:
+     .;lOXK0000KNMMMMX00000KO;     ;KMMMMMNl.   .;ccl:,.
+     .;:c:'.....kMMMNo........    'OMMMWMMMK:    '::;;'.
+   .......     .xMMMNl           .dWMMXdOMMMO'   ........
+   .:cc:;.     .xMMMNc          .lNMMNo.:XMMWx.    .:cl:.
+   .:llc,.     .:xxxd,          ;KMMMk. .oWMMNl.   .:llc'
+   .cll:.     .;:;;:::,.       'OMMMK:';''kWMMK:   .;llc,
+   .cll:.     .,;;;;;;,.     .,xWMMNl.:l:.;KMMMO'  .;llc'
+   .:llc.      .cOOOk;      .lKNMMWx..:l:..lNMMWx. .:llc'
+   .;lcc,.     .xMMMNc      :KMMMM0, .:lc. .xWMMNl.'ccl:.
+    .cllc.     .xMMMNc     'OMMMMXc...:lc...,0MMMKl:lcc,.
+    .,ccl:.    .xMMMNc    .xWMMMWo.,;;:lc;;;.cXMMMXdcc;.
+     .,clc:.   .xMMMNc   .lNMMMWk. .':clc:,. .dWMMW0o;.
+      .,clcc,. .ckkkx;   .okkkOx,    .';,.    'kKKK0l.
+       .':lcc:'.....      .  ..            ..,;cllc,.
+         .,cclc,....                     ....;clc;..
+          ..,:,..,c:'..              ...';:,..,:,.
+            ....:lcccc:;,'''.....'',;;:clllc,....
+               .'',;:cllllllccccclllllcc:,'..
+                   ...'',,;;;;;;;;;,''...
+                            .....
+
+"""
+
+TOPIC = """
+#######################
+#  Environment Setup  #
+#######################
+
+"""
+
+EXPECTED_ENV_VARS = [
+    "TA_USERNAME",
+    "TA_PASSWORD",
+    "ELASTIC_PASSWORD",
+    "ES_URL",
+    "TA_HOST",
+]
+INST = "https://github.com/tubearchivist/tubearchivist#installing-and-updating"
+NGINX = "/etc/nginx/sites-available/default"
+UWSGI = "/app/uwsgi.ini"
+
+
+class Command(BaseCommand):
+    """command framework"""
+
+    # pylint: disable=no-member
+    help = "Check environment before startup"
+
+    def handle(self, *args, **options):
+        """run all commands"""
+        self.stdout.write(LOGO)
+        self.stdout.write(TOPIC)
+        self._expected_vars()
+        self._elastic_user_overwrite()
+        self._ta_port_overwrite()
+        self._ta_uwsgi_overwrite()
+        self._enable_cast_overwrite()
+        self._create_superuser()
+
+    def _expected_vars(self):
+        """check if expected env vars are set"""
+        self.stdout.write("[1] checking expected env vars")
+        env = os.environ
+        for var in EXPECTED_ENV_VARS:
+            if not env.get(var):
+                message = f"    🗙 expected env var {var} not set\n    {INST}"
+                self.stdout.write(self.style.ERROR(message))
+                raise CommandError(message)
+
+        message = "    ✓ all expected env vars are set"
+        self.stdout.write(self.style.SUCCESS(message))
+
+    def _elastic_user_overwrite(self):
+        """check for ELASTIC_USER overwrite"""
+        self.stdout.write("[2] set default ES user")
+        if not os.environ.get("ELASTIC_USER"):
+            os.environ.setdefault("ELASTIC_USER", "elastic")
+
+        env = os.environ.get("ELASTIC_USER")
+
+        self.stdout.write(self.style.SUCCESS(f"    ✓ ES user is set to {env}"))
+
+    def _ta_port_overwrite(self):
+        """set TA_PORT overwrite for nginx"""
+        self.stdout.write("[3] check TA_PORT overwrite")
+        overwrite = os.environ.get("TA_PORT")
+        if not overwrite:
+            self.stdout.write(self.style.SUCCESS("    TA_PORT is not set"))
+            return
+
+        regex = re.compile(r"listen [0-9]{1,5}")
+        changed = file_overwrite(NGINX, regex, overwrite)
+        if changed:
+            message = f"    ✓ TA_PORT changed to {overwrite}"
+        else:
+            message = f"    ✓ TA_PORT already set to {overwrite}"
+
+        self.stdout.write(self.style.SUCCESS(message))
+
+    def _ta_uwsgi_overwrite(self):
+        """set TA_UWSGI_PORT overwrite"""
+        self.stdout.write("[4] check TA_UWSGI_PORT overwrite")
+        overwrite = os.environ.get("TA_UWSGI_PORT")
+        if not overwrite:
+            message = "    TA_UWSGI_PORT is not set"
+            self.stdout.write(self.style.SUCCESS(message))
+            return
+
+        # nginx
+        regex = re.compile(r"uwsgi_pass localhost:[0-9]{1,5}")
+        to_overwrite = f"uwsgi_pass localhost:{overwrite}"
+        changed = file_overwrite(NGINX, regex, to_overwrite)
+
+        # uwsgi
+        regex = re.compile(r"socket = :[0-9]{1,5}")
+        to_overwrite = f"socket = :{overwrite}"
+        changed = file_overwrite(UWSGI, regex, to_overwrite)
+
+        if changed:
+            message = f"    ✓ TA_UWSGI_PORT changed to {overwrite}"
+        else:
+            message = f"    ✓ TA_UWSGI_PORT already set to {overwrite}"
+
+        self.stdout.write(self.style.SUCCESS(message))
+
+    def _enable_cast_overwrite(self):
+        """cast workaround, remove auth for static files in nginx"""
+        self.stdout.write("[5] check ENABLE_CAST overwrite")
+        overwrite = os.environ.get("ENABLE_CAST")
+        if not overwrite:
+            self.stdout.write(self.style.SUCCESS("    ENABLE_CAST is not set"))
+            return
+
+        regex = re.compile(r"[^\S\r\n]*auth_request /api/ping/;\n")
+        changed = file_overwrite(NGINX, regex, "")
+        if changed:
+            message = "    ✓ process nginx to enable Cast"
+        else:
+            message = "    ✓ Cast is already enabled in nginx"
+
+        self.stdout.write(self.style.SUCCESS(message))
+
+    def _create_superuser(self):
+        """create superuser if not exist"""
+        self.stdout.write("[6] create superuser")
+        is_created = Account.objects.filter(is_superuser=True)
+        if is_created:
+            message = "    superuser already created"
+            self.stdout.write(self.style.SUCCESS(message))
+            return
+
+        name = os.environ.get("TA_USERNAME")
+        password = os.environ.get("TA_PASSWORD")
+        Account.objects.create_superuser(name, password)
+        message = f"    ✓ new superuser with name {name} created"
+        self.stdout.write(self.style.SUCCESS(message))
+
+
+def file_overwrite(file_path, regex, overwrite):
+    """change file content from old to overwrite, return true when changed"""
+    with open(file_path, "r", encoding="utf-8") as f:
+        file_content = f.read()
+
+    changed = re.sub(regex, overwrite, file_content)
+    if changed == file_content:
+        return False
+
+    with open(file_path, "w", encoding="utf-8") as f:
+        f.write(changed)
+
+    return True

tubearchivist/config/settings.py

@@ -58,6 +58,7 @@ INSTALLED_APPS = [
     "rest_framework",
     "rest_framework.authtoken",
     "api",
+    "config",
 ]
 
 MIDDLEWARE = [

tubearchivist/home/src/es/connect.py

@@ -32,15 +32,15 @@ class ElasticWrap:
         self.auth = self.config["application"]["es_auth"]
         self.url = f"{es_url}/{self.path}"
 
-    def get(self, data=False):
+    def get(self, data=False, timeout=10, print_error=True):
         """get data from es"""
         if data:
             response = requests.get(
-                self.url, json=data, auth=self.auth, timeout=10
+                self.url, json=data, auth=self.auth, timeout=timeout
             )
         else:
-            response = requests.get(self.url, auth=self.auth, timeout=10)
-        if not response.ok:
+            response = requests.get(self.url, auth=self.auth, timeout=timeout)
+        if print_error and not response.ok:
             print(response.text)
 
         return response.json(), response.status_code