From cacf6e43b8fcd374f747b88cfecfb675f7a141fe Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 28 Jan 2023 08:37:58 +0700 Subject: [PATCH] add envcheck and basic connection check startup command --- docker_assets/run.sh | 73 ++----- tubearchivist/config/management/__init__.py | 0 .../config/management/commands/__init__.py | 0 .../management/commands/ta_connection.py | 57 +++++ .../config/management/commands/ta_envcheck.py | 195 ++++++++++++++++++ tubearchivist/config/settings.py | 1 + tubearchivist/home/src/es/connect.py | 8 +- 7 files changed, 273 insertions(+), 61 deletions(-) create mode 100644 tubearchivist/config/management/__init__.py create mode 100644 tubearchivist/config/management/commands/__init__.py create mode 100644 tubearchivist/config/management/commands/ta_connection.py create mode 100644 tubearchivist/config/management/commands/ta_envcheck.py diff --git a/docker_assets/run.sh b/docker_assets/run.sh index e4bdf1f..ca1fdd9 100644 --- a/docker_assets/run.sh +++ b/docker_assets/run.sh @@ -1,71 +1,30 @@ #!/bin/bash # startup script inside the container for tubearchivist -if [[ -z "$ELASTIC_USER" ]]; then - export ELASTIC_USER=elastic -fi -cachedir=/cache -[[ -d $cachedir ]] || cachedir=. -lockfile=${cachedir}/initsu.lock - -required="Missing required environment variable" -[[ -f $lockfile ]] || : "${TA_USERNAME:?$required}" -: "${TA_PASSWORD:?$required}" -: "${ELASTIC_PASSWORD:?$required}" -: "${TA_HOST:?$required}" - -# ugly nginx and uwsgi port overwrite with env vars -if [[ -n "$TA_PORT" ]]; then - sed -i "s/8000/$TA_PORT/g" /etc/nginx/sites-available/default -fi - -if [[ -n "$TA_UWSGI_PORT" ]]; then - sed -i "s/8080/$TA_UWSGI_PORT/g" /etc/nginx/sites-available/default - sed -i "s/8080/$TA_UWSGI_PORT/g" /app/uwsgi.ini -fi - -# disable auth on static files for cast support -if [[ -n "$ENABLE_CAST" ]]; then - sed -i "/auth_request/d" /etc/nginx/sites-available/default -fi +# check environment +python manage.py ta_envcheck +python manage.py ta_connection # wait for elasticsearch -counter=0 -until curl -u "$ELASTIC_USER":"$ELASTIC_PASSWORD" "$ES_URL" -fs; do - echo "waiting for elastic search to start" - counter=$((counter+1)) - if [[ $counter -eq 12 ]]; then - # fail after 2 min - echo "failed to connect to elastic search, exiting..." - curl -v -u "$ELASTIC_USER":"$ELASTIC_PASSWORD" "$ES_URL"?pretty - exit 1 - fi - sleep 10 -done +# counter=0 +# until curl -u "$ELASTIC_USER":"$ELASTIC_PASSWORD" "$ES_URL" -fs; do +# echo "waiting for elastic search to start" +# counter=$((counter+1)) +# if [[ $counter -eq 12 ]]; then +# # fail after 2 min +# echo "failed to connect to elastic search, exiting..." +# curl -v -u "$ELASTIC_USER":"$ELASTIC_PASSWORD" "$ES_URL"?pretty +# exit 1 +# fi +# sleep 10 +# done # start python application python manage.py makemigrations python manage.py migrate +# python manage.py collectstatic --noinput -c -if [[ -f $lockfile ]]; then - echo -e "\e[33;1m[WARNING]\e[0m This is not the first run! Skipping" \ - "superuser creation.\nTo force it, remove $lockfile" -else - export DJANGO_SUPERUSER_PASSWORD=$TA_PASSWORD - output="$(python manage.py createsuperuser --noinput --name "$TA_USERNAME" 2>&1)" - - case "$output" in - *"Superuser created successfully"*) - echo "$output" && touch $lockfile ;; - *"That name is already taken."*) - echo "Superuser already exists. Creation will be skipped on next start." - touch $lockfile ;; - *) echo "$output" && exit 1 - esac -fi - -python manage.py collectstatic --noinput -c nginx & celery -A home.tasks worker --loglevel=INFO & celery -A home beat --loglevel=INFO \ diff --git a/tubearchivist/config/management/__init__.py b/tubearchivist/config/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tubearchivist/config/management/commands/__init__.py b/tubearchivist/config/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tubearchivist/config/management/commands/ta_connection.py b/tubearchivist/config/management/commands/ta_connection.py new file mode 100644 index 0000000..9c68821 --- /dev/null +++ b/tubearchivist/config/management/commands/ta_connection.py @@ -0,0 +1,57 @@ +""" +Functionality: +- check that all connections are working +""" + +import sys +from time import sleep + +import requests +from django.core.management.base import BaseCommand, CommandError +from home.src.es.connect import ElasticWrap + + +TOPIC = """ + +####################### +# Connection check # +####################### + +""" + +class Command(BaseCommand): + """command framework""" + + TIMEOUT = 120 + + # pylint: disable=no-member + help = "Check connections" + + def handle(self, *args, **options): + """run all commands""" + self.stdout.write(TOPIC) + self._es_connection_check() + + def _es_connection_check(self): + """wait for elasticsearch connection""" + self.stdout.write("[1] connect to Elastic Search") + sys.stdout.write(" .") + for _ in range(self.TIMEOUT // 5): + sleep(2) + sys.stdout.write(".") + try: + response, status_code = ElasticWrap("/").get( + timeout=1, print_error=False + ) + except requests.exceptions.ConnectionError: + pass + + if status_code == 200: + self.stdout.write("\n ✓ ES connection established") + return + + message = " 🗙 ES connection failed" + self.stdout.write(self.style.ERROR(f"\n{message}")) + self.stdout.write(f" error message: {response | None}") + self.stdout.write(f" status code: {status_code | None}") + raise CommandError(message) diff --git a/tubearchivist/config/management/commands/ta_envcheck.py b/tubearchivist/config/management/commands/ta_envcheck.py new file mode 100644 index 0000000..a0f216d --- /dev/null +++ b/tubearchivist/config/management/commands/ta_envcheck.py @@ -0,0 +1,195 @@ +""" +Functionality: +- Check environment at startup +- Process config file overwrites from env var +- Stop startup on error +- python management.py ta_envcheck +""" + +import os +import re + +from django.core.management.base import BaseCommand, CommandError +from home.models import Account + +LOGO = """ + + .... ..... + ...'',;:cc,. .;::;;,'... + ..,;:cccllclc, .:ccllllcc;,.. + ..,:cllcc:;,'.',. ....'',;ccllc:,.. + ..;cllc:,'.. ...,:cccc:'. + .;cccc;.. ..,:ccc:'. + .ckkkOkxollllllllllllc. .,:::;. .,cclc; + .:0MMMMMMMMMMMMMMMMMMMX: .cNMMMWx. .;clc: + .;lOXK0000KNMMMMX00000KO; ;KMMMMMNl. .;ccl:,. + .;:c:'.....kMMMNo........ 'OMMMWMMMK: '::;;'. + ....... .xMMMNl .dWMMXdOMMMO' ........ + .:cc:;. .xMMMNc .lNMMNo.:XMMWx. .:cl:. + .:llc,. .:xxxd, ;KMMMk. .oWMMNl. .:llc' + .cll:. .;:;;:::,. 'OMMMK:';''kWMMK: .;llc, + .cll:. .,;;;;;;,. .,xWMMNl.:l:.;KMMMO' .;llc' + .:llc. .cOOOk; .lKNMMWx..:l:..lNMMWx. .:llc' + .;lcc,. .xMMMNc :KMMMM0, .:lc. .xWMMNl.'ccl:. + .cllc. .xMMMNc 'OMMMMXc...:lc...,0MMMKl:lcc,. + .,ccl:. .xMMMNc .xWMMMWo.,;;:lc;;;.cXMMMXdcc;. + .,clc:. .xMMMNc .lNMMMWk. .':clc:,. .dWMMW0o;. + .,clcc,. .ckkkx; .okkkOx, .';,. 'kKKK0l. + .':lcc:'..... . .. ..,;cllc,. + .,cclc,.... ....;clc;.. + ..,:,..,c:'.. ...';:,..,:,. + ....:lcccc:;,'''.....'',;;:clllc,.... + .'',;:cllllllccccclllllcc:,'.. + ...'',,;;;;;;;;;,''... + ..... + +""" + +TOPIC = """ +####################### +# Environment Setup # +####################### + +""" + +EXPECTED_ENV_VARS = [ + "TA_USERNAME", + "TA_PASSWORD", + "ELASTIC_PASSWORD", + "ES_URL", + "TA_HOST", +] +INST = "https://github.com/tubearchivist/tubearchivist#installing-and-updating" +NGINX = "/etc/nginx/sites-available/default" +UWSGI = "/app/uwsgi.ini" + + +class Command(BaseCommand): + """command framework""" + + # pylint: disable=no-member + help = "Check environment before startup" + + def handle(self, *args, **options): + """run all commands""" + self.stdout.write(LOGO) + self.stdout.write(TOPIC) + self._expected_vars() + self._elastic_user_overwrite() + self._ta_port_overwrite() + self._ta_uwsgi_overwrite() + self._enable_cast_overwrite() + self._create_superuser() + + def _expected_vars(self): + """check if expected env vars are set""" + self.stdout.write("[1] checking expected env vars") + env = os.environ + for var in EXPECTED_ENV_VARS: + if not env.get(var): + message = f" 🗙 expected env var {var} not set\n {INST}" + self.stdout.write(self.style.ERROR(message)) + raise CommandError(message) + + message = " ✓ all expected env vars are set" + self.stdout.write(self.style.SUCCESS(message)) + + def _elastic_user_overwrite(self): + """check for ELASTIC_USER overwrite""" + self.stdout.write("[2] set default ES user") + if not os.environ.get("ELASTIC_USER"): + os.environ.setdefault("ELASTIC_USER", "elastic") + + env = os.environ.get("ELASTIC_USER") + + self.stdout.write(self.style.SUCCESS(f" ✓ ES user is set to {env}")) + + def _ta_port_overwrite(self): + """set TA_PORT overwrite for nginx""" + self.stdout.write("[3] check TA_PORT overwrite") + overwrite = os.environ.get("TA_PORT") + if not overwrite: + self.stdout.write(self.style.SUCCESS(" TA_PORT is not set")) + return + + regex = re.compile(r"listen [0-9]{1,5}") + changed = file_overwrite(NGINX, regex, overwrite) + if changed: + message = f" ✓ TA_PORT changed to {overwrite}" + else: + message = f" ✓ TA_PORT already set to {overwrite}" + + self.stdout.write(self.style.SUCCESS(message)) + + def _ta_uwsgi_overwrite(self): + """set TA_UWSGI_PORT overwrite""" + self.stdout.write("[4] check TA_UWSGI_PORT overwrite") + overwrite = os.environ.get("TA_UWSGI_PORT") + if not overwrite: + message = " TA_UWSGI_PORT is not set" + self.stdout.write(self.style.SUCCESS(message)) + return + + # nginx + regex = re.compile(r"uwsgi_pass localhost:[0-9]{1,5}") + to_overwrite = f"uwsgi_pass localhost:{overwrite}" + changed = file_overwrite(NGINX, regex, to_overwrite) + + # uwsgi + regex = re.compile(r"socket = :[0-9]{1,5}") + to_overwrite = f"socket = :{overwrite}" + changed = file_overwrite(UWSGI, regex, to_overwrite) + + if changed: + message = f" ✓ TA_UWSGI_PORT changed to {overwrite}" + else: + message = f" ✓ TA_UWSGI_PORT already set to {overwrite}" + + self.stdout.write(self.style.SUCCESS(message)) + + def _enable_cast_overwrite(self): + """cast workaround, remove auth for static files in nginx""" + self.stdout.write("[5] check ENABLE_CAST overwrite") + overwrite = os.environ.get("ENABLE_CAST") + if not overwrite: + self.stdout.write(self.style.SUCCESS(" ENABLE_CAST is not set")) + return + + regex = re.compile(r"[^\S\r\n]*auth_request /api/ping/;\n") + changed = file_overwrite(NGINX, regex, "") + if changed: + message = " ✓ process nginx to enable Cast" + else: + message = " ✓ Cast is already enabled in nginx" + + self.stdout.write(self.style.SUCCESS(message)) + + def _create_superuser(self): + """create superuser if not exist""" + self.stdout.write("[6] create superuser") + is_created = Account.objects.filter(is_superuser=True) + if is_created: + message = " superuser already created" + self.stdout.write(self.style.SUCCESS(message)) + return + + name = os.environ.get("TA_USERNAME") + password = os.environ.get("TA_PASSWORD") + Account.objects.create_superuser(name, password) + message = f" ✓ new superuser with name {name} created" + self.stdout.write(self.style.SUCCESS(message)) + + +def file_overwrite(file_path, regex, overwrite): + """change file content from old to overwrite, return true when changed""" + with open(file_path, "r", encoding="utf-8") as f: + file_content = f.read() + + changed = re.sub(regex, overwrite, file_content) + if changed == file_content: + return False + + with open(file_path, "w", encoding="utf-8") as f: + f.write(changed) + + return True diff --git a/tubearchivist/config/settings.py b/tubearchivist/config/settings.py index cda4cfc..e200b8b 100644 --- a/tubearchivist/config/settings.py +++ b/tubearchivist/config/settings.py @@ -58,6 +58,7 @@ INSTALLED_APPS = [ "rest_framework", "rest_framework.authtoken", "api", + "config", ] MIDDLEWARE = [ diff --git a/tubearchivist/home/src/es/connect.py b/tubearchivist/home/src/es/connect.py index 5f674d6..4b59ace 100644 --- a/tubearchivist/home/src/es/connect.py +++ b/tubearchivist/home/src/es/connect.py @@ -32,15 +32,15 @@ class ElasticWrap: self.auth = self.config["application"]["es_auth"] self.url = f"{es_url}/{self.path}" - def get(self, data=False): + def get(self, data=False, timeout=10, print_error=True): """get data from es""" if data: response = requests.get( - self.url, json=data, auth=self.auth, timeout=10 + self.url, json=data, auth=self.auth, timeout=timeout ) else: - response = requests.get(self.url, auth=self.auth, timeout=10) - if not response.ok: + response = requests.get(self.url, auth=self.auth, timeout=timeout) + if print_error and not response.ok: print(response.text) return response.json(), response.status_code