chore: Automatic reloading of uwsgi in docker environment.

This makes uwsgi reload when changes are detected for python or static files in developer-only environment.
2024-03-08 18:46:18 +01:00 · 2024-03-08 18:46:18 +01:00 · 51bc36c76c
parent 8778546577
commit 51bc36c76c
8 changed files with 162 additions and 26 deletions
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@ -126,19 +126,17 @@ The documentation available at [docs.tubearchivist.com](https://docs.tubearchivi

 I have learned the hard way, that working on a dockerized application outside of docker is very error prone and in general not a good idea. So if you want to test your changes, it's best to run them in a docker testing environment. You might be able to run the application directly, but this document assumes you're using docker.

-### Instructions
-
-Set up docker on your development machine.
-
-Clone this repository.
+### Instructions (Docker)

 Functional changes should be made against the unstable `testing` branch, so check that branch out, then make a new branch for your work.
+There is two ways to run the application, either in a docker container as described below, or in a VM. The VM is a bit more complex to set up, but it's a good way to simulate a low end environment.

-Edit the `docker-compose.yml` file and replace the [`image: bbilly1/tubearchivist` line](https://github.com/tubearchivist/tubearchivist/blob/4af12aee15620e330adf3624c984c3acf6d0ac8b/docker-compose.yml#L7) with `build: .`. Also make any other changes to the environment variables and so on necessary to run the application, just like you're launching the application as normal.
+1. Set up docker on your development machine.
+2. Clone this repository.
+3. Optionally edit the `docker-compose.dev.yml` file to your liking or leave it as is. Especially the `TA_HOST` which is the domain name of the application that needs to match the URL accessing the application i.e `127.0.0.1`.
+4. Run `make dev` which will start the the docker compose project with the `docker-compose.dev.yml` file. Kill it with `ctrl-c` or by running `docker compose down` from a new terminal window in the same directory.
+5. Make your changes, application will automatically reload.

-Run `docker compose up --build`. This will bring up the application. Kill it with `ctrl-c` or by running `docker compose down` from a new terminal window in the same directory.
-
-Make your changes locally and re-run `docker compose up --build`. The `Dockerfile` is structured in a way that the actual application code is in the last layer so rebuilding the image with only code changes utilizes the build cache for everything else and will just take a few seconds.

 ### Develop environment inside a VM

--- a/34
+++ b/34
@ -35,16 +35,16 @@ RUN apt-get clean && apt-get -y update && apt-get -y install --no-install-recomm
 # install patched ffmpeg build, default to linux64
 RUN if [ "$TARGETPLATFORM" = "linux/arm64" ] ; then \
    curl -s https://api.github.com/repos/yt-dlp/FFmpeg-Builds/releases/latest \
-        | grep browser_download_url \
-        | grep ".*master.*linuxarm64.*tar.xz" \
-        | cut -d '"' -f 4 \
-        | xargs curl -L --output ffmpeg.tar.xz ; \
+    | grep browser_download_url \
+    | grep ".*master.*linuxarm64.*tar.xz" \
+    | cut -d '"' -f 4 \
+    | xargs curl -L --output ffmpeg.tar.xz ; \
    else \
    curl -s https://api.github.com/repos/yt-dlp/FFmpeg-Builds/releases/latest \
-        | grep browser_download_url \
-        | grep ".*master.*linux64.*tar.xz" \
-        | cut -d '"' -f 4 \
-        | xargs curl -L --output ffmpeg.tar.xz ; \
+    | grep browser_download_url \
+    | grep ".*master.*linux64.*tar.xz" \
+    | cut -d '"' -f 4 \
+    | xargs curl -L --output ffmpeg.tar.xz ; \
    fi && \
    tar -xf ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/bin/ "ffmpeg" && \
    tar -xf ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/bin/ "ffprobe" && \
@ -52,9 +52,9 @@ RUN if [ "$TARGETPLATFORM" = "linux/arm64" ] ; then \

 # install debug tools for testing environment
 RUN if [ "$INSTALL_DEBUG" ] ; then \
-        apt-get -y update && apt-get -y install --no-install-recommends \
-        vim htop bmon net-tools iputils-ping procps \
-        && pip install --user ipython \
+    apt-get -y update && apt-get -y install --no-install-recommends \
+    vim htop bmon net-tools iputils-ping procps \
+    && pip install --user ipython \
    ; fi

 # make folders
@ -65,9 +65,11 @@ COPY docker_assets/nginx.conf /etc/nginx/sites-available/default
 RUN sed -i 's/^user www\-data\;$/user root\;/' /etc/nginx/nginx.conf

 # copy application into container
-COPY ./tubearchivist /app
-COPY ./docker_assets/run.sh /app
-COPY ./docker_assets/uwsgi.ini /app
+#COPY ./tubearchivist /app
+COPY ./docker_assets/run-dev.sh /
+COPY ./docker_assets/run.sh /
+COPY ./docker_assets/uwsgi-dev.ini /
+COPY ./docker_assets/uwsgi.ini /

 # volumes
 VOLUME /cache
@ -77,6 +79,6 @@ VOLUME /youtube
 WORKDIR /app
 EXPOSE 8000

-RUN chmod +x ./run.sh
+RUN chmod +x /run.sh /run-dev.sh

-CMD ["./run.sh"]
+ENTRYPOINT [ "/run.sh" ]
--- a/20
+++ b/20
@ -0,0 +1,20 @@
+DOCKER_COMPOSE := docker-compose
+DOCKER_COMPOSE_PROD := $(DOCKER_COMPOSE) -f docker-compose.yml
+DOCKER_COMPOSE_DEV := $(DOCKER_COMPOSE) -f docker-compose.dev.yml
+
+.PHONY: help prod dev clean
+
+help:
+	@echo "Usage: make [TARGET]"
+	@echo "Targets:"
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "  \033[36m%-8s\033[0m %s\n", $$1, $$2}'
+
+prod:
+	$(DOCKER_COMPOSE_PROD) up -d
+
+dev:
+	$(DOCKER_COMPOSE_DEV) up --build --force-recreate
+
+clean:
+	$(DOCKER_COMPOSE_PROD) down -v --remove-orphans
+	$(DOCKER_COMPOSE_DEV) down -v --remove-orphans
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@ -0,0 +1,67 @@
+version: '3.5'
+
+services:
+  tubearchivist:
+    container_name: tubearchivist
+    restart: unless-stopped
+    image: tubearchivist
+    build:
+      context: .
+      dockerfile: Dockerfile
+      args:
+        - INSTALL_DEBUG=true
+    entrypoint: /run-dev.sh
+    ports:
+      - 8000:8000
+    volumes:
+      - ./tubearchivist:/app
+      - media:/youtube
+      - cache:/cache
+    environment:
+      - ES_URL=http://archivist-es:9200     # needs protocol e.g. http and port
+      - REDIS_HOST=archivist-redis          # don't add protocol
+      - HOST_UID=1000
+      - HOST_GID=1000
+      - TA_HOST=tubearchivist.local         # set your host name
+      - TA_USERNAME=tubearchivist           # your initial TA credentials
+      - TA_PASSWORD=verysecret              # your initial TA credentials
+      - ELASTIC_PASSWORD=verysecret         # set password for Elasticsearch
+      - TZ=America/New_York                 # set your time zone
+      - DJANGO_DEBUG=true
+    depends_on:
+      - archivist-es
+      - archivist-redis
+  archivist-redis:
+    image: redis/redis-stack-server
+    container_name: archivist-redis
+    restart: unless-stopped
+    expose:
+      - "6379"
+    volumes:
+      - redis:/data
+    depends_on:
+      - archivist-es
+  archivist-es:
+    image: bbilly1/tubearchivist-es # only for amd64, or use official es 8.12.0
+    container_name: archivist-es
+    restart: unless-stopped
+    environment:
+      - "ELASTIC_PASSWORD=verysecret" # matching Elasticsearch password
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+      - "xpack.security.enabled=true"
+      - "discovery.type=single-node"
+      - "path.repo=/usr/share/elasticsearch/data/snapshot"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    volumes:
+      - es:/usr/share/elasticsearch/data # check for permission error when using bind mount, see readme
+    expose:
+      - "9200"
+
+volumes:
+  media:
+  cache:
+  redis:
+  es:
--- a/docker_assets/run-dev.sh
+++ b/docker_assets/run-dev.sh
@ -0,0 +1,31 @@
+#!/bin/bash
+# startup script inside the container for tubearchivist
+
+set -e
+
+cd /app || exit 1
+
+# django setup
+python manage.py migrate
+
+if [[ -z "$DJANGO_DEBUG" ]]; then
+    python manage.py collectstatic --noinput -c
+fi
+
+# ta setup
+python manage.py ta_envcheck
+python manage.py ta_connection
+python manage.py ta_startup
+python manage.py ta_migpath
+
+#export PYTHONWARNINGS=error
+
+pip install watchdog -U
+watchmedo shell-command --patterns="*.html;*.css;*.js" --recursive --command='kill -HUP `cat /tmp/project-master.pid`' . &
+
+# start all tasks
+nginx &
+celery -A home.tasks worker --loglevel=INFO --max-tasks-per-child 10 &
+celery -A home beat --loglevel=INFO \
+    -s "${BEAT_SCHEDULE_PATH:-${cachedir}/celerybeat-schedule}" &
+uwsgi --ini /uwsgi-dev.ini
--- a/docker_assets/run.sh
+++ b/docker_assets/run.sh
@ -3,6 +3,8 @@

 set -e

+cd /app || exit 1
+
 # django setup
 python manage.py migrate

@ -21,4 +23,4 @@ nginx &
 celery -A home.tasks worker --loglevel=INFO --max-tasks-per-child 10 &
 celery -A home beat --loglevel=INFO \
    -s "${BEAT_SCHEDULE_PATH:-${cachedir}/celerybeat-schedule}" &
-uwsgi --ini uwsgi.ini
+uwsgi --ini /uwsgi.ini
--- a/docker_assets/uwsgi-dev.ini
+++ b/docker_assets/uwsgi-dev.ini
@ -0,0 +1,15 @@
+[uwsgi]
+module = config.wsgi:application
+chdir = /app
+master = True
+pidfile = /tmp/project-master.pid
+vacuum = True
+max-requests = 5000
+socket = :8080
+buffer-size = 8192
+log-5xx = true
+log-4xx = true
+processes = 1
+threads = 2
+enable-threads = False
+py-autoreload = 3
--- a/docker_assets/uwsgi.ini
+++ b/docker_assets/uwsgi.ini
@ -1,5 +1,6 @@
 [uwsgi]
 module = config.wsgi:application
+chdir = /app
 master = True
 pidfile = /tmp/project-master.pid
 vacuum = True