♻️ Simplify Docker Compose files and deployment (#153)

* ♻️ Simplify Docker Compose files and deployment

* 🔧 Remove TRAEFIK_PUBLIC_NETWORK_IS_EXTERNAL from .env

{{cookiecutter.project_slug}}/.env

@@ -1,6 +1,3 @@
-COMPOSE_PATH_SEPARATOR=:
-COMPOSE_FILE=docker-compose.shared.admin.yml:docker-compose.shared.base-images.yml:docker-compose.shared.depends.yml:docker-compose.shared.env.yml:docker-compose.dev.build.yml:docker-compose.dev.command.yml:docker-compose.dev.env.yml:docker-compose.dev.labels.yml:docker-compose.dev.networks.yml:docker-compose.dev.ports.yml:docker-compose.dev.volumes.yml
-
 DOMAIN=localhost
 # DOMAIN=local.dockertoolbox.tiangolo.com
 # DOMAIN=localhost.tiangolo.com

{{cookiecutter.project_slug}}/docker-compose.deploy.build.yml

@@ -1,15 +0,0 @@
-version: '3.3'
-services:
-  backend:
-    build: 
-      context: ./backend
-      dockerfile: backend.dockerfile
-  celeryworker:
-    build:
-      context: ./backend
-      dockerfile: celeryworker.dockerfile
-  frontend:
-    build: 
-      context: ./frontend
-      args:
-          FRONTEND_ENV: ${FRONTEND_ENV-production}

{{cookiecutter.project_slug}}/docker-compose.deploy.command.yml

@@ -1,11 +0,0 @@
-version: '3.3'
-services:
-  proxy:
-    command: --docker \
-      --docker.swarmmode \
-      --docker.watch \
-      --docker.exposedbydefault=false \
-      --constraints=tag==${TRAEFIK_TAG} \
-      --logLevel=INFO \
-      --accessLog \
-      --web

{{cookiecutter.project_slug}}/docker-compose.deploy.images.yml

@@ -1,8 +0,0 @@
-version: '3.3'
-services:
-  backend:
-    image: '${DOCKER_IMAGE_BACKEND}:${TAG-latest}'
-  celeryworker:
-    image: '${DOCKER_IMAGE_CELERYWORKER}:${TAG-latest}'
-  frontend:
-    image: '${DOCKER_IMAGE_FRONTEND}:${TAG-latest}'

{{cookiecutter.project_slug}}/docker-compose.deploy.labels.yml

@@ -1,57 +0,0 @@
-version: '3.3'
-services:
-  pgadmin:
-    deploy:
-      labels:
-        - traefik.frontend.rule=Host:pgadmin.${DOMAIN}
-        - traefik.enable=true
-        - traefik.port=5050
-        - traefik.tags=${TRAEFIK_PUBLIC_TAG}
-        - traefik.docker.network=${TRAEFIK_PUBLIC_NETWORK}
-        - traefik.frontend.entryPoints=http,https
-        - traefik.frontend.redirect.entryPoint=https
-  proxy:
-    deploy:
-      labels:
-        # For the configured domain
-        - traefik.frontend.rule=Host:${DOMAIN}
-        # For a domain with and without 'www'
-        # Comment the previous line above and un-comment the line below
-        # - "traefik.frontend.rule=Host:www.${DOMAIN},${DOMAIN}"
-        - traefik.enable=true
-        - traefik.port=80
-        - traefik.tags=${TRAEFIK_PUBLIC_TAG}
-        - traefik.docker.network=${TRAEFIK_PUBLIC_NETWORK}
-        - traefik.frontend.entryPoints=http,https
-        - traefik.frontend.redirect.entryPoint=https
-        # Uncomment the config line below to detect and redirect www to non-www (or the contrary)
-        # The lines above for traefik.frontend.rule are needed too
-        # - "traefik.frontend.redirect.regex=^https?://(www.)?(${DOMAIN})/(.*)"
-        # To redirect from non-www to www un-comment the line below
-        # - "traefik.frontend.redirect.replacement=https://www.${DOMAIN}/$$3"
-        # To redirect from www to non-www un-comment the line below
-        # - "traefik.frontend.redirect.replacement=https://${DOMAIN}/$$3"
-  flower:
-    deploy:
-      labels:
-        - traefik.frontend.rule=Host:flower.${DOMAIN}
-        - traefik.enable=true
-        - traefik.port=5555
-        - traefik.tags=${TRAEFIK_PUBLIC_TAG}
-        - traefik.docker.network=${TRAEFIK_PUBLIC_NETWORK}
-        - traefik.frontend.entryPoints=http,https
-        - traefik.frontend.redirect.entryPoint=https
-  backend:
-    deploy:
-      labels:
-        - traefik.frontend.rule=PathPrefix:/api,/docs,/redoc
-        - traefik.enable=true
-        - traefik.port=80
-        - traefik.tags=${TRAEFIK_TAG}
-  frontend:
-    deploy:
-      labels:
-        - traefik.frontend.rule=PathPrefix:/
-        - traefik.enable=true
-        - traefik.port=80
-        - traefik.tags=${TRAEFIK_TAG}

{{cookiecutter.project_slug}}/docker-compose.deploy.networks.yml

@@ -1,18 +0,0 @@
-version: '3.3'
-services:
-  pgadmin:
-    networks:
-      - ${TRAEFIK_PUBLIC_NETWORK}
-      - default
-  proxy:
-    networks:
-      - ${TRAEFIK_PUBLIC_NETWORK}
-      - default
-  flower:
-    networks:
-      - ${TRAEFIK_PUBLIC_NETWORK}
-      - default
-
-networks:
-  traefik-public:
-    external: true

{{cookiecutter.project_slug}}/docker-compose.deploy.volumes-placement.yml

@@ -1,17 +0,0 @@
-version: '3.3'
-services:
-  db:
-    volumes:
-      - app-db-data:/var/lib/postgresql/data/pgdata
-    deploy:
-      placement:
-        constraints:
-          - node.labels.${STACK_NAME}.app-db-data == true
-  proxy:
-    deploy:
-      placement:
-        constraints:
-          - node.role == manager
-
-volumes:
-  app-db-data:

{{cookiecutter.project_slug}}/docker-compose.dev.build.yml

@@ -1,19 +0,0 @@
-version: '3.3'
-services:
-  backend:
-    build: 
-      context: ./backend
-      dockerfile: backend.dockerfile
-      args: 
-        env: dev
-  celeryworker:
-    build:
-      context: ./backend
-      dockerfile: celeryworker.dockerfile
-      args: 
-        env: dev
-  frontend:
-    build: 
-      context: ./frontend
-      args: 
-        FRONTEND_ENV: dev

{{cookiecutter.project_slug}}/docker-compose.dev.command.yml

@@ -1,14 +0,0 @@
-version: '3.3'
-services:
-  proxy:
-    command: --docker \
-      --docker.watch \
-      --docker.exposedbydefault=false \
-      --constraints=tag==${TRAEFIK_TAG} \
-      --logLevel=DEBUG \
-      --accessLog \
-      --web
-  # backend:
-  #   command: bash -c "while true; do sleep 1; done"  # Infinite loop to keep container live doing nothing
-  backend:
-    command: /start-reload.sh

{{cookiecutter.project_slug}}/docker-compose.dev.env.yml

@@ -1,11 +0,0 @@
-version: '3.3'
-services:
-  backend:
-    environment:
-      - JUPYTER=jupyter lab --ip=0.0.0.0 --allow-root --NotebookApp.custom_display_url=http://127.0.0.1:8888
-      - SERVER_HOST=http://${DOMAIN}
-  celeryworker:
-    environment:
-      - RUN=celery worker -A app.worker -l info -Q main-queue -c 1
-      - JUPYTER=jupyter lab --ip=0.0.0.0 --allow-root --NotebookApp.custom_display_url=http://127.0.0.1:8888
-      - SERVER_HOST=http://${DOMAIN}

{{cookiecutter.project_slug}}/docker-compose.dev.labels.yml

@@ -1,19 +0,0 @@
-version: '3.3'
-services:
-  proxy:
-    labels:
-      - traefik.frontend.rule=Host:${DOMAIN}
-      - traefik.enable=true
-      - traefik.port=80
-  backend:
-    labels:
-      - traefik.frontend.rule=PathPrefix:/api,/docs,/redoc
-      - traefik.enable=true
-      - traefik.port=80
-      - traefik.tags=${TRAEFIK_TAG}
-  frontend:
-    labels:
-      - traefik.frontend.rule=PathPrefix:/
-      - traefik.enable=true
-      - traefik.port=80
-      - traefik.tags=${TRAEFIK_TAG}

{{cookiecutter.project_slug}}/docker-compose.dev.networks.yml

@@ -1,7 +0,0 @@
-version: '3.3'
-services:
-  backend:
-    networks: 
-      default: 
-        aliases: 
-          - ${DOMAIN}

{{cookiecutter.project_slug}}/docker-compose.dev.ports.yml

@@ -1,15 +0,0 @@
-version: '3.3'
-services:
-  pgadmin:
-    ports:
-      - '5050:5050'
-  proxy:
-    ports:
-      - '80:80'
-      - '8090:8080'
-  flower:
-    ports:
-      - '5555:5555'
-  backend:
-    ports:
-      - '8888:8888'

{{cookiecutter.project_slug}}/docker-compose.dev.volumes.yml

@@ -1,8 +0,0 @@
-version: '3.3'
-services:
-  backend:
-    volumes:
-      - ./backend/app:/app
-  celeryworker:
-    volumes:
-      - ./backend/app:/app

{{cookiecutter.project_slug}}/docker-compose.override.yml

@@ -0,0 +1,76 @@
+version: "3.3"
+services:
+
+  proxy:
+    ports:
+      - "80:80"
+      - "8090:8080"
+    command: --docker \
+      --docker.watch \
+      --docker.exposedbydefault=false \
+      --constraints=tag==${TRAEFIK_TAG} \
+      --logLevel=DEBUG \
+      --accessLog \
+      --web
+    labels:
+      - traefik.frontend.rule=Host:${DOMAIN}
+      - traefik.enable=true
+      - traefik.port=80
+
+  pgadmin:
+    ports:
+      - "5050:5050"
+
+  flower:
+    ports:
+      - "5555:5555"
+
+  backend:
+    ports:
+      - "8888:8888"
+    volumes:
+      - ./backend/app:/app
+    environment:
+      - JUPYTER=jupyter lab --ip=0.0.0.0 --allow-root --NotebookApp.custom_display_url=http://127.0.0.1:8888
+      - SERVER_HOST=http://${DOMAIN}
+    build:
+      context: ./backend
+      dockerfile: backend.dockerfile
+      args:
+        env: dev
+    # command: bash -c "while true; do sleep 1; done"  # Infinite loop to keep container live doing nothing
+    command: /start-reload.sh
+    labels:
+      - traefik.frontend.rule=PathPrefix:/api,/docs,/redoc
+      - traefik.enable=true
+      - traefik.port=80
+      - traefik.tags=${TRAEFIK_TAG}
+
+  celeryworker:
+    volumes:
+      - ./backend/app:/app
+    environment:
+      - RUN=celery worker -A app.worker -l info -Q main-queue -c 1
+      - JUPYTER=jupyter lab --ip=0.0.0.0 --allow-root --NotebookApp.custom_display_url=http://127.0.0.1:8888
+      - SERVER_HOST=http://${DOMAIN}
+    build:
+      context: ./backend
+      dockerfile: celeryworker.dockerfile
+      args:
+        env: dev
+
+  frontend:
+    build:
+      context: ./frontend
+      args:
+        FRONTEND_ENV: dev
+    labels:
+      - traefik.frontend.rule=PathPrefix:/
+      - traefik.enable=true
+      - traefik.port=80
+      - traefik.tags=${TRAEFIK_TAG}
+
+networks:
+  traefik-public:
+    # For local dev, don't expect an external Traefik network
+    external: false

{{cookiecutter.project_slug}}/docker-compose.shared.admin.yml

@@ -1,21 +0,0 @@
-version: '3.3'
-services:
-  pgadmin:
-    image: dpage/pgadmin4
-    depends_on:
-      - db
-    env_file:
-      - .env
-  proxy:
-    image: traefik:v1.7
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
-  flower:
-    image: mher/flower
-    env_file:
-      - .env
-    command:
-      - "--broker=amqp://guest@queue:5672//"
-      # For the "Broker" tab to work in the flower UI, uncomment the following command argument,
-      # and change the queue service's image as described in docker-compose.shared.base-images.yml
-      # - "--broker_api=http://guest:guest@queue:15672/api//"

{{cookiecutter.project_slug}}/docker-compose.shared.base-images.yml

@@ -1,10 +0,0 @@
-version: '3.3'
-services:
-  db:
-    image: postgres:12
-  queue:
-    image: rabbitmq:3
-    # Using the below image instead is required to enable the "Broker" tab in the flower UI:
-    # image: rabbitmq:3-management
-    #
-    # You also have to change the flower command as documented in docker-compose.shared.admin.yml

{{cookiecutter.project_slug}}/docker-compose.shared.depends.yml

@@ -1,9 +0,0 @@
-version: '3.3'
-services:
-  backend:
-    depends_on: 
-      - db
-  celeryworker:
-    depends_on: 
-      - db
-      - queue

{{cookiecutter.project_slug}}/docker-compose.shared.env.yml

@@ -1,23 +0,0 @@
-version: '3.3'
-services:
-  db:
-    env_file:
-      - .env
-    environment:
-      - PGDATA=/var/lib/postgresql/data/pgdata
-  backend:
-    env_file:
-      - .env
-    environment:
-      - SERVER_NAME=${DOMAIN}
-      - SERVER_HOST=https://${DOMAIN}
-      # Allow explicit env var override for tests
-      - SMTP_HOST=${SMTP_HOST}
-  celeryworker:
-    env_file:
-      - .env
-    environment:
-      - SERVER_NAME=${DOMAIN}
-      - SERVER_HOST=https://${DOMAIN}
-      # Allow explicit env var override for tests
-      - SMTP_HOST=${SMTP_HOST}

{{cookiecutter.project_slug}}/docker-compose.yml

@@ -0,0 +1,160 @@
+version: "3.3"
+services:
+
+  proxy:
+    image: traefik:v1.7
+    networks:
+      - ${TRAEFIK_PUBLIC_NETWORK}
+      - default
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    command: --docker \
+      --docker.swarmmode \
+      --docker.watch \
+      --docker.exposedbydefault=false \
+      --constraints=tag==${TRAEFIK_TAG} \
+      --logLevel=INFO \
+      --accessLog \
+      --web
+    deploy:
+      placement:
+        constraints:
+          - node.role == manager
+      labels:
+        # For the configured domain
+        - traefik.frontend.rule=Host:${DOMAIN}
+        # For a domain with and without 'www'
+        # Comment the previous line above and un-comment the line below
+        # - "traefik.frontend.rule=Host:www.${DOMAIN},${DOMAIN}"
+        - traefik.enable=true
+        - traefik.port=80
+        - traefik.tags=${TRAEFIK_PUBLIC_TAG}
+        - traefik.docker.network=${TRAEFIK_PUBLIC_NETWORK}
+        - traefik.frontend.entryPoints=http,https
+        - traefik.frontend.redirect.entryPoint=https
+        # Uncomment the config line below to detect and redirect www to non-www (or the contrary)
+        # The lines above for traefik.frontend.rule are needed too
+        # - "traefik.frontend.redirect.regex=^https?://(www.)?(${DOMAIN})/(.*)"
+        # To redirect from non-www to www un-comment the line below
+        # - "traefik.frontend.redirect.replacement=https://www.${DOMAIN}/$$3"
+        # To redirect from www to non-www un-comment the line below
+        # - "traefik.frontend.redirect.replacement=https://${DOMAIN}/$$3"
+  
+  db:
+    image: postgres:12
+    volumes:
+      - app-db-data:/var/lib/postgresql/data/pgdata
+    env_file:
+      - .env
+    environment:
+      - PGDATA=/var/lib/postgresql/data/pgdata
+    deploy:
+      placement:
+        constraints:
+          - node.labels.${STACK_NAME}.app-db-data == true
+
+  pgadmin:
+    image: dpage/pgadmin4
+    networks:
+      - ${TRAEFIK_PUBLIC_NETWORK}
+      - default
+    depends_on:
+      - db
+    env_file:
+      - .env
+    deploy:
+      labels:
+        - traefik.frontend.rule=Host:pgadmin.${DOMAIN}
+        - traefik.enable=true
+        - traefik.port=5050
+        - traefik.tags=${TRAEFIK_PUBLIC_TAG}
+        - traefik.docker.network=${TRAEFIK_PUBLIC_NETWORK}
+        - traefik.frontend.entryPoints=http,https
+        - traefik.frontend.redirect.entryPoint=https
+
+  queue:
+    image: rabbitmq:3
+    # Using the below image instead is required to enable the "Broker" tab in the flower UI:
+    # image: rabbitmq:3-management
+    #
+    # You also have to change the flower command
+  
+  flower:
+    image: mher/flower
+    networks:
+      - ${TRAEFIK_PUBLIC_NETWORK}
+      - default
+    env_file:
+      - .env
+    command:
+      - "--broker=amqp://guest@queue:5672//"
+      # For the "Broker" tab to work in the flower UI, uncomment the following command argument,
+      # and change the queue service's image as well
+      # - "--broker_api=http://guest:guest@queue:15672/api//"
+    deploy:
+      labels:
+        - traefik.frontend.rule=Host:flower.${DOMAIN}
+        - traefik.enable=true
+        - traefik.port=5555
+        - traefik.tags=${TRAEFIK_PUBLIC_TAG}
+        - traefik.docker.network=${TRAEFIK_PUBLIC_NETWORK}
+        - traefik.frontend.entryPoints=http,https
+        - traefik.frontend.redirect.entryPoint=https
+  
+  backend:
+    image: '${DOCKER_IMAGE_BACKEND}:${TAG-latest}'
+    depends_on:
+      - db
+    env_file:
+      - .env
+    environment:
+      - SERVER_NAME=${DOMAIN}
+      - SERVER_HOST=https://${DOMAIN}
+      # Allow explicit env var override for tests
+      - SMTP_HOST=${SMTP_HOST}
+    build:
+      context: ./backend
+      dockerfile: backend.dockerfile
+    deploy:
+      labels:
+        - traefik.frontend.rule=PathPrefix:/api,/docs,/redoc
+        - traefik.enable=true
+        - traefik.port=80
+        - traefik.tags=${TRAEFIK_TAG}
+  
+  celeryworker:
+    image: '${DOCKER_IMAGE_CELERYWORKER}:${TAG-latest}'
+    depends_on:
+      - db
+      - queue
+    env_file:
+      - .env
+    environment:
+      - SERVER_NAME=${DOMAIN}
+      - SERVER_HOST=https://${DOMAIN}
+      # Allow explicit env var override for tests
+      - SMTP_HOST=${SMTP_HOST}
+    build:
+      context: ./backend
+      dockerfile: celeryworker.dockerfile
+  
+  frontend:
+    image: '${DOCKER_IMAGE_FRONTEND}:${TAG-latest}'
+    build:
+      context: ./frontend
+      args:
+        FRONTEND_ENV: ${FRONTEND_ENV-production}
+    deploy:
+      labels:
+        - traefik.frontend.rule=PathPrefix:/
+        - traefik.enable=true
+        - traefik.port=80
+        - traefik.tags=${TRAEFIK_TAG}
+
+volumes:
+  app-db-data:
+
+networks:
+  traefik-public:
+    # Allow setting it to false for testing
+    external: ${TRAEFIK_PUBLIC_NETWORK_IS_EXTERNAL-true}

{{cookiecutter.project_slug}}/scripts/build-push.sh

@@ -7,4 +7,4 @@ TAG=${TAG} \
 FRONTEND_ENV=${FRONTEND_ENV-production} \
 . ./scripts/build.sh
 
-docker-compose -f docker-stack.yml push
+docker-compose -f docker-compose.yml push

{{cookiecutter.project_slug}}/scripts/build.sh

@@ -6,8 +6,5 @@ set -e
 TAG=${TAG} \
 FRONTEND_ENV=${FRONTEND_ENV-production} \
 docker-compose \
--f docker-compose.deploy.build.yml \
--f docker-compose.deploy.images.yml \
-config > docker-stack.yml
-
-docker-compose -f docker-stack.yml build
+-f docker-compose.yml \
+build

{{cookiecutter.project_slug}}/scripts/deploy.sh

@@ -8,17 +8,9 @@ TRAEFIK_TAG=${TRAEFIK_TAG} \
 STACK_NAME=${STACK_NAME} \
 TAG=${TAG} \
 docker-compose \
--f docker-compose.shared.admin.yml \
--f docker-compose.shared.base-images.yml \
--f docker-compose.shared.depends.yml \
--f docker-compose.shared.env.yml \
--f docker-compose.deploy.command.yml \
--f docker-compose.deploy.images.yml \
--f docker-compose.deploy.labels.yml \
--f docker-compose.deploy.networks.yml \
--f docker-compose.deploy.volumes-placement.yml \
+-f docker-compose.yml \
 config > docker-stack.yml
 
 docker-auto-labels docker-stack.yml
 
-docker stack deploy -c docker-stack.yml --with-registry-auth ${STACK_NAME}
+docker stack deploy -c docker-stack.yml --with-registry-auth "${STACK_NAME}"

{{cookiecutter.project_slug}}/scripts/test-local.sh

@@ -3,27 +3,13 @@
 # Exit in case of error
 set -e
 
+docker-compose down -v --remove-orphans # Remove possibly previous broken stacks left hanging after an error
+
 if [ $(uname -s) = "Linux" ]; then
     echo "Remove __pycache__ files"
     sudo find . -type d -name __pycache__ -exec rm -r {} \+
 fi
 
-docker-compose \
-    -f docker-compose.shared.admin.yml \
-    -f docker-compose.shared.base-images.yml \
-    -f docker-compose.shared.depends.yml \
-    -f docker-compose.shared.env.yml \
-    -f docker-compose.dev.build.yml \
-    -f docker-compose.dev.env.yml \
-    -f docker-compose.dev.labels.yml \
-    -f docker-compose.dev.networks.yml \
-    -f docker-compose.dev.ports.yml \
-    -f docker-compose.dev.volumes.yml \
-    config > docker-stack.yml
-
-#    -f docker-compose.dev.command.yml \
-
-docker-compose -f docker-stack.yml build
-docker-compose -f docker-stack.yml down -v --remove-orphans # Remove possibly previous broken stacks left hanging after an error
-docker-compose -f docker-stack.yml up -d
-docker-compose -f docker-stack.yml exec -T backend-tests /tests-start.sh "$@"
+docker-compose build
+docker-compose up -d
+docker-compose exec -T backend bash /app/tests-start.sh "$@"

{{cookiecutter.project_slug}}/scripts/test.sh

@@ -3,12 +3,11 @@
 # Exit in case of error
 set -e
 
-DOMAIN=backend SMTP_HOST="" \
+DOMAIN=backend \
+SMTP_HOST="" \
+TRAEFIK_PUBLIC_NETWORK_IS_EXTERNAL=false \
 docker-compose \
--f docker-compose.shared.base-images.yml \
--f docker-compose.shared.env.yml \
--f docker-compose.shared.depends.yml \
--f docker-compose.deploy.build.yml \
+-f docker-compose.yml \
 config > docker-stack.yml
 
 docker-compose -f docker-stack.yml build