Replace Caddy with Nginx

.circleci/config.yml

@@ -31,6 +31,7 @@ jobs:
       - run:
           name: "Setup: Create directories for MinIO (cannot be made by docker for some reason)"
           command: |
+            echo "127.0.0.1 minio" | sudo tee -a /etc/hosts
             mkdir -p var/minio/public
             mkdir -p var/minio/private
 
@@ -89,6 +90,8 @@ jobs:
           path: tests/test-results
       - store_artifacts:
           path: dockerLogs/
+      - store_artifacts:
+          path: var/log/
 
 workflows:
   version: 2

.env_circleci

@@ -8,12 +8,13 @@ DB_PORT=5432
 
 RABBITMQ_DEFAULT_USER=rabbit-username
 RABBITMQ_DEFAULT_PASS=rabbit-password-you-should-change
+RABBITMQ_MANAGEMENT_PORT=15672
 RABBITMQ_PORT=5672
 RABBITMQ_HOST=rabbit
 WORKER_CONNECTION_TIMEOUT=100000000 # milliseconds
 
 FLOWER_BASIC_AUTH=root:password-you-should-change
-
+FLOWER_PUBLIC_PORT=5555
 DJANGO_SETTINGS_MODULE=settings.test
 
 # Minio local storage example
@@ -28,11 +29,18 @@ AWS_SECRET_ACCESS_KEY=testsecret
 AWS_STORAGE_BUCKET_NAME=public
 AWS_STORAGE_PRIVATE_BUCKET_NAME=private
 # NOTE! port 9000 here should match $MINIO_PORT
-AWS_S3_ENDPOINT_URL=http://172.17.0.1:9000/
+AWS_S3_ENDPOINT_URL=http://minio:9000/
 AWS_QUERYSTRING_AUTH=False
 DJANGO_SUPERUSER_PASSWORD=codabench
 DJANGO_SUPERUSER_EMAIL=test@test.com
 DJANGO_SUPERUSER_USERNAME=codabench
-DOMAIN_NAME=localhost:80
 TLS_EMAIL=your@email.com
-SUBMISSIONS_API_URL=http://django:8000/api
+SUBMISSIONS_API_URL=http://django:8000/api
+
+# -----------------------------------------------------------------------------
+# Nginx settings
+# -----------------------------------------------------------------------------
+HTTPS=False
+RATE_LIMIT=100
+DOMAIN_NAME=localhost
+DATABASE_ACCESS=True

.env_sample

@@ -1,42 +1,55 @@
-SECRET_KEY=change-this-secret
+# Use openssl rand -hex 32 to generate this secret key, or generate it however you want and copy it here
+SECRET_KEY=
 
 # For local setup and debug
-DEBUG=True
+DEBUG=False
 
+# -----------------------------------------------------------------------------
 # Database
+# -----------------------------------------------------------------------------
 DB_HOST=db
 DB_NAME=postgres
 DB_USERNAME=postgres
 DB_PASSWORD=postgres
 DB_PORT=5432
 
+# -----------------------------------------------------------------------------
+# Django
+# -----------------------------------------------------------------------------
 DJANGO_SETTINGS_MODULE=settings.develop
-ALLOWED_HOSTS=localhost,example.com
+ALLOWED_HOSTS=localhost,
 SUBMISSIONS_API_URL=http://django:8000/api
 MAX_EXECUTION_TIME_LIMIT=600 # time limit for the default queue (in seconds)
 
-# Local domain definition
-DOMAIN_NAME=localhost:80
-
-# SSL style domain definition
+# -----------------------------------------------------------------------------
+# Nginx settings
+# -----------------------------------------------------------------------------
+HTTPS=False
+RATE_LIMIT=5
+DOMAIN_NAME=localhost
 TLS_EMAIL=your@email.com
-# DOMAIN_NAME=example.com:443
 
+# -----------------------------------------------------------------------------
+# RabbitMQ
+# -----------------------------------------------------------------------------
 RABBITMQ_HOST=rabbit
 RABBITMQ_DEFAULT_USER=rabbit-username
 RABBITMQ_DEFAULT_PASS=rabbit-password-you-should-change
 RABBITMQ_MANAGEMENT_PORT=15672
 RABBITMQ_PORT=5672
 WORKER_CONNECTION_TIMEOUT=100000000 # milliseconds
-#RABBITMQ_HTTP_PROXY=http://proxy-example:3128
-#RABBITMQ_HTTPS_PROXY=http://proxy-example:3128
-#RABBITMQ_NO_PROXY=localhost,172.0.0.0/8
 
-FLOWER_PUBLIC_PORT=5555
 
+# -----------------------------------------------------------------------------
+# Flower
+# -----------------------------------------------------------------------------
+FLOWER_PUBLIC_PORT=5555
 FLOWER_BASIC_AUTH=root:password-you-should-change
 
-SELENIUM_HOSTNAME=selenium
+
+# -----------------------------------------------------------------------------
+# Email Settings
+# -----------------------------------------------------------------------------
 
 # Uncomment to enable email settings
 #EMAIL_BACKEND=django.core.mail.backends.smtp.EmailBackend
@@ -74,21 +87,6 @@ AWS_QUERYSTRING_AUTH=False
 #WORKER_BUNDLE_URL_REWRITE=http://localhost:9000|http://minio:9000
 
 
-# -----------------------------------------------------------------------------
-# Limit for re-running submission
-# This is used to limit users to rerun submissions
-# on default queue when number of submissions are < RERUN_SUBMISSION_LIMIT
-# -----------------------------------------------------------------------------
-RERUN_SUBMISSION_LIMIT=30
-
-
-# -----------------------------------------------------------------------------
-# Enable or disbale regular email sign-in an sign-up
-# -----------------------------------------------------------------------------
-ENABLE_SIGN_UP=True
-ENABLE_SIGN_IN=True
-
-
 # # S3 storage example
 # STORAGE_TYPE=s3
 # AWS_ACCESS_KEY_ID=12312312312312312331223
@@ -111,6 +109,20 @@ ENABLE_SIGN_IN=True
 # GS_PRIVATE_BUCKET_NAME=private
 # GOOGLE_APPLICATION_CREDENTIALS=/app/certs/google-storage-api.json
 
+# -----------------------------------------------------------------------------
+# Limit for re-running submission
+# This is used to limit users to rerun submissions
+# on default queue when number of submissions are < RERUN_SUBMISSION_LIMIT
+# -----------------------------------------------------------------------------
+RERUN_SUBMISSION_LIMIT=30
+
+
+# -----------------------------------------------------------------------------
+# Enable or disbale regular email sign-in an sign-up
+# -----------------------------------------------------------------------------
+ENABLE_SIGN_UP=True
+ENABLE_SIGN_IN=True
+
 
 # -----------------------------------------------------------------------------
 # Logging (Serialized outputs the logs in JSON format)

.gitignore

@@ -43,3 +43,5 @@ caddy_data/
 home_page_counters.json
 my-postgres.conf
 tests/config/state.json
+state.json
+certs/

docker-compose.yml

@@ -2,24 +2,31 @@ services:
   #----------------------------------------------------------------------------------------------------
   #   Web Services
   #----------------------------------------------------------------------------------------------------
-  caddy:
-    image: caddy:2.10.0
+  nginx:
+    image: nginx:alpine
     env_file: .env
     environment:
-      - ACME_AGREE=true
+      - NGINX_ENVSUBST_OUTPUT_DIR=/etc/nginx
+    command: ["nginx", "-g", "daemon off;"]
     volumes:
-      - ./Caddyfile:/etc/caddy/Caddyfile
-      - ./src/staticfiles:/var/www/django/static
-      - ./caddy_data:/data
-      - ./caddy_config:/config
-      - ./var/log/caddy:/var/log/
-      - ./maintenance_mode/:/srv
+      - ./src/staticfiles:/var/www/django/static:ro
+      - ./maintenance_mode/:/srv:ro
+      - ./nginx/:/etc/nginx/templates/:ro
+      - ./nginx/certs/:/var/cache/nginx/acme-letsencrypt
+      - ./var/log/nginx/:/var/log/nginx
     restart: unless-stopped
     ports:
       - 80:80
       - 443:443
-    depends_on:
-      - django
+      - 8000:8000
+      - 5432:5432
+      - ${RABBITMQ_MANAGEMENT_PORT:-15672}:15672
+      - ${RABBITMQ_PORT}:5672
+      - ${MINIO_PORT:-9000}:9000
+      - ${FLOWER_PUBLIC_PORT:-5555}:5555
+    networks:
+      - frontend
+      - backend
 
   django:
     build: 
@@ -37,7 +44,7 @@ services:
       - ./var/logs:/app/logs
     restart: unless-stopped
     ports:
-      - 8000:8000
+      - 8000
     depends_on:
       - db
       - rabbit
@@ -48,24 +55,32 @@ services:
       options:
         max-size: "20m"
         max-file: "5"
+    networks:
+      - backend
+      - frontend
 
 
   #----------------------------------------------------------------------------------------------------
   # Minio local storage helper
   #----------------------------------------------------------------------------------------------------
   minio:
     image: minio/minio:RELEASE.2025-04-22T22-12-26Z
-    command: server /export
+    command: server /export --console-address ":9001"
     volumes:
       - ./var/minio:/export
     restart: unless-stopped
     ports:
-      - $MINIO_PORT:9000
+      - 9000
+      - 9001
     env_file: .env
+    environment:
+      MINIO_BROWSER_REDIRECT_URL: "http://${DOMAIN_NAME}/console"
     healthcheck:
       test: ["CMD", "curl", "-I", "http://minio:9000/minio/health/live"]
       interval: 5s
       retries: 5
+    networks:
+      - backend
   createbuckets:
     image: minio/mc:RELEASE.2025-07-21T05-28-08Z
     depends_on:
@@ -90,6 +105,8 @@ services:
       fi;
       exit 0;
       "
+    networks:
+      - backend
 
   #----------------------------------------------------------------------------------------------------
   #   Local development helper, rebuilds RiotJS/Stylus on change
@@ -106,7 +123,6 @@ services:
         max-size: "20m"
         max-file: "5"
 
-
   #----------------------------------------------------------------------------------------------------
   #   Database Service
   #
@@ -121,16 +137,18 @@ services:
       - POSTGRES_PASSWORD=${DB_PASSWORD}
     command: ["postgres", "-c", "log_statement=all", "-c", "log_destination=stderr", "-c", "config_file=/etc/postgresql/postgresql.conf"]
     ports:
-      - 5432:5432
+      - 5432
     volumes:
       - ./var/postgres:/var/lib/postgresql/18/:delegated
       - ./backups:/app/backups
-      - ./my-postgres.conf:/etc/postgresql/postgresql.conf
+      - ./my-postgres.conf:/etc/postgresql/postgresql.conf:ro
     restart: unless-stopped
     logging:
       options:
         max-size: "20m"
         max-file: "5"
+    networks:
+      - backend
 
   #----------------------------------------------------------------------------------------------------
   #   Rabbitmq & Flower monitoring tool
@@ -145,20 +163,18 @@ services:
     # containers being destroyed..!
     hostname: rabbit
     env_file: .env
-    environment:
-      - http_proxy=${RABBITMQ_HTTP_PROXY}
-      - https_proxy=${RABBITMQ_HTTPS_PROXY}
-      - no_proxy=${RABBITMQ_NO_PROXY}
     ports:
-      - ${RABBITMQ_MANAGEMENT_PORT:-15672}:15672
-      - ${RABBITMQ_PORT}:5672
+      - 15672
+      - 5672
     volumes:
       - ./var/rabbit:/var/lib/rabbitmq
     restart: unless-stopped
     logging:
       options:
         max-size: "20m"
         max-file: "5"
+    networks:
+      - backend
 
   flower:
     image: mher/flower
@@ -167,26 +183,30 @@ services:
       - CELERY_BROKER_URL=pyamqp://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@${RABBITMQ_HOST}:${RABBITMQ_PORT}//
     restart: unless-stopped
     ports:
-      - ${FLOWER_PUBLIC_PORT:-5555}:5555
+      - 5555
     depends_on:
       - rabbit
     logging:
         options:
             max-size: "20m"
             max-file: "5"
+    networks:
+      - backend
 
   #----------------------------------------------------------------------------------------------------
   #   Redis
   #----------------------------------------------------------------------------------------------------
   redis:
     image: redis
     ports:
-      - 6379:6379
+      - 6379
     restart: unless-stopped
     logging:
       options:
         max-size: "20m"
         max-file: "5"
+    networks:
+      - backend
 
   #----------------------------------------------------------------------------------------------------
   #   Celery Service
@@ -215,6 +235,8 @@ services:
           # Limit memory substantially here so we see any problems that may
           # appear on Heroku ahead of time
           memory: 256M
+    networks:
+      - backend
 
   compute_worker:
     command: ["celery -A compute_worker worker -l info -Q compute-worker -n compute-worker@%n"]
@@ -226,7 +248,7 @@ services:
       - django
       - rabbit
     volumes:
-      - ./compute_worker:/app
+      - ./compute_worker:/app:ro
       - ${HOST_DIRECTORY:-/tmp/codabench}:/codabench
       # Actual connection back to docker parent to run things
       - /var/run/docker.sock:/var/run/docker.sock
@@ -242,3 +264,11 @@ services:
       options:
         max-size: "20m"
         max-file: "5"
+    networks:
+      - backend
+      - frontend
+
+networks:
+  frontend:
+  backend:
+    internal: true