From 26ce5431f02e1802f36d8c63f0ff0e2b4710888c Mon Sep 17 00:00:00 2001 From: Obada Haddad Date: Mon, 2 Mar 2026 18:07:53 +0100 Subject: [PATCH 1/6] replace caddy with nginx allowing us to route all connection through it (rabbit,minio etc) --- .env_circleci | 15 +++- .env_sample | 14 +-- docker-compose.yml | 69 +++++++++++---- nginx/extra/anti_scrapper.conf.template | 19 ++++ nginx/extra/base_django.conf.template | 23 +++++ nginx/extra/block_map.conf.template | 33 +++++++ nginx/extra/maintenance.conf.template | 5 ++ nginx/http/flower.conf.template | 7 ++ nginx/http/minio.conf.template | 32 +++++++ nginx/http/rabbit_console.conf.template | 6 ++ nginx/nginx.conf.template | 113 ++++++++++++++++++++++++ nginx/robots.txt | 2 + nginx/stream/database.conf.template | 4 + nginx/stream/rabbit.conf.template | 4 + tests/pytest.ini | 2 +- tests/test_auth.py | 2 +- 16 files changed, 319 insertions(+), 31 deletions(-) create mode 100644 nginx/extra/anti_scrapper.conf.template create mode 100644 nginx/extra/base_django.conf.template create mode 100644 nginx/extra/block_map.conf.template create mode 100644 nginx/extra/maintenance.conf.template create mode 100644 nginx/http/flower.conf.template create mode 100644 nginx/http/minio.conf.template create mode 100644 nginx/http/rabbit_console.conf.template create mode 100644 nginx/nginx.conf.template create mode 100644 nginx/robots.txt create mode 100644 nginx/stream/database.conf.template create mode 100644 nginx/stream/rabbit.conf.template diff --git a/.env_circleci b/.env_circleci index f56adf935..d66c0b8e5 100644 --- a/.env_circleci +++ b/.env_circleci @@ -8,12 +8,13 @@ DB_PORT=5432 RABBITMQ_DEFAULT_USER=rabbit-username RABBITMQ_DEFAULT_PASS=rabbit-password-you-should-change +RABBITMQ_MANAGEMENT_PORT=15672 RABBITMQ_PORT=5672 RABBITMQ_HOST=rabbit WORKER_CONNECTION_TIMEOUT=100000000 # milliseconds FLOWER_BASIC_AUTH=root:password-you-should-change - +FLOWER_PUBLIC_PORT=5555 DJANGO_SETTINGS_MODULE=settings.test # Minio local storage example @@ -28,11 +29,17 @@ AWS_SECRET_ACCESS_KEY=testsecret AWS_STORAGE_BUCKET_NAME=public AWS_STORAGE_PRIVATE_BUCKET_NAME=private # NOTE! port 9000 here should match $MINIO_PORT -AWS_S3_ENDPOINT_URL=http://172.17.0.1:9000/ +AWS_S3_ENDPOINT_URL=http://minio:9000/ AWS_QUERYSTRING_AUTH=False DJANGO_SUPERUSER_PASSWORD=codabench DJANGO_SUPERUSER_EMAIL=test@test.com DJANGO_SUPERUSER_USERNAME=codabench -DOMAIN_NAME=localhost:80 TLS_EMAIL=your@email.com -SUBMISSIONS_API_URL=http://django:8000/api \ No newline at end of file +SUBMISSIONS_API_URL=http://django:8000/api + +# ----------------------------------------------------------------------------- +# Nginx settings +# ----------------------------------------------------------------------------- +HTTPS=False +RATE_LIMIT=100 +DOMAIN_NAME=localhost diff --git a/.env_sample b/.env_sample index 6bd01cfbd..854c8f9d3 100644 --- a/.env_sample +++ b/.env_sample @@ -1,7 +1,7 @@ SECRET_KEY=change-this-secret # For local setup and debug -DEBUG=True +DEBUG=False # Database DB_HOST=db @@ -15,8 +15,13 @@ ALLOWED_HOSTS=localhost,example.com SUBMISSIONS_API_URL=http://django:8000/api MAX_EXECUTION_TIME_LIMIT=600 # time limit for the default queue (in seconds) -# Local domain definition -DOMAIN_NAME=localhost:80 +# ----------------------------------------------------------------------------- +# Nginx settings +# ----------------------------------------------------------------------------- +HTTPS=False +RATE_LIMIT=5 +DOMAIN_NAME=localhost + # SSL style domain definition TLS_EMAIL=your@email.com @@ -28,9 +33,6 @@ RABBITMQ_DEFAULT_PASS=rabbit-password-you-should-change RABBITMQ_MANAGEMENT_PORT=15672 RABBITMQ_PORT=5672 WORKER_CONNECTION_TIMEOUT=100000000 # milliseconds -#RABBITMQ_HTTP_PROXY=http://proxy-example:3128 -#RABBITMQ_HTTPS_PROXY=http://proxy-example:3128 -#RABBITMQ_NO_PROXY=localhost,172.0.0.0/8 FLOWER_PUBLIC_PORT=5555 diff --git a/docker-compose.yml b/docker-compose.yml index 46c1783de..9c5f0f88b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,24 +2,37 @@ services: #---------------------------------------------------------------------------------------------------- # Web Services #---------------------------------------------------------------------------------------------------- - caddy: - image: caddy:2.10.0 + nginx: + image: nginx:latest env_file: .env environment: - - ACME_AGREE=true + - NGINX_ENVSUBST_OUTPUT_DIR=/etc/nginx + command: ["nginx", "-g", "daemon off;"] volumes: - - ./Caddyfile:/etc/caddy/Caddyfile - ./src/staticfiles:/var/www/django/static - - ./caddy_data:/data - - ./caddy_config:/config - - ./var/log/caddy:/var/log/ - ./maintenance_mode/:/srv + - ./certsNginx:/var/cache/nginx/acme-letsencrypt + - ./nginx/:/etc/nginx/templates/ + - ./var/log/nginx/:/var/log/nginx restart: unless-stopped ports: - 80:80 - 443:443 + - 8000:8000 + - 5432:5432 + - ${RABBITMQ_MANAGEMENT_PORT:-15672}:15672 + - ${RABBITMQ_PORT}:5672 + - ${MINIO_PORT}:9000 + - ${FLOWER_PUBLIC_PORT:-5555}:5555 depends_on: - django + - minio + - rabbit + - site_worker + - db + networks: + - frontend + - backend django: build: @@ -37,7 +50,7 @@ services: - ./var/logs:/app/logs restart: unless-stopped ports: - - 8000:8000 + - 8000 depends_on: - db - rabbit @@ -48,6 +61,8 @@ services: options: max-size: "20m" max-file: "5" + networks: + - backend #---------------------------------------------------------------------------------------------------- @@ -60,12 +75,14 @@ services: - ./var/minio:/export restart: unless-stopped ports: - - $MINIO_PORT:9000 + - 9000 env_file: .env healthcheck: test: ["CMD", "curl", "-I", "http://minio:9000/minio/health/live"] interval: 5s retries: 5 + networks: + - backend createbuckets: image: minio/mc:RELEASE.2025-07-21T05-28-08Z depends_on: @@ -90,6 +107,8 @@ services: fi; exit 0; " + networks: + - backend #---------------------------------------------------------------------------------------------------- # Local development helper, rebuilds RiotJS/Stylus on change @@ -106,7 +125,6 @@ services: max-size: "20m" max-file: "5" - #---------------------------------------------------------------------------------------------------- # Database Service # @@ -121,7 +139,7 @@ services: - POSTGRES_PASSWORD=${DB_PASSWORD} command: ["postgres", "-c", "log_statement=all", "-c", "log_destination=stderr", "-c", "config_file=/etc/postgresql/postgresql.conf"] ports: - - 5432:5432 + - 5432 volumes: - ./var/postgres:/var/lib/postgresql/18/:delegated - ./backups:/app/backups @@ -131,6 +149,8 @@ services: options: max-size: "20m" max-file: "5" + networks: + - backend #---------------------------------------------------------------------------------------------------- # Rabbitmq & Flower monitoring tool @@ -145,13 +165,9 @@ services: # containers being destroyed..! hostname: rabbit env_file: .env - environment: - - http_proxy=${RABBITMQ_HTTP_PROXY} - - https_proxy=${RABBITMQ_HTTPS_PROXY} - - no_proxy=${RABBITMQ_NO_PROXY} ports: - - ${RABBITMQ_MANAGEMENT_PORT:-15672}:15672 - - ${RABBITMQ_PORT}:5672 + - 15672 + - 5672 volumes: - ./var/rabbit:/var/lib/rabbitmq restart: unless-stopped @@ -159,6 +175,8 @@ services: options: max-size: "20m" max-file: "5" + networks: + - backend flower: image: mher/flower @@ -167,13 +185,15 @@ services: - CELERY_BROKER_URL=pyamqp://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@${RABBITMQ_HOST}:${RABBITMQ_PORT}// restart: unless-stopped ports: - - ${FLOWER_PUBLIC_PORT:-5555}:5555 + - 5555 depends_on: - rabbit logging: options: max-size: "20m" max-file: "5" + networks: + - backend #---------------------------------------------------------------------------------------------------- # Redis @@ -181,12 +201,14 @@ services: redis: image: redis ports: - - 6379:6379 + - 6379 restart: unless-stopped logging: options: max-size: "20m" max-file: "5" + networks: + - backend #---------------------------------------------------------------------------------------------------- # Celery Service @@ -215,6 +237,8 @@ services: # Limit memory substantially here so we see any problems that may # appear on Heroku ahead of time memory: 256M + networks: + - backend compute_worker: command: ["celery -A compute_worker worker -l info -Q compute-worker -n compute-worker@%n"] @@ -242,3 +266,10 @@ services: options: max-size: "20m" max-file: "5" + networks: + - backend + +networks: + frontend: + backend: + internal: true diff --git a/nginx/extra/anti_scrapper.conf.template b/nginx/extra/anti_scrapper.conf.template new file mode 100644 index 000000000..d60c26751 --- /dev/null +++ b/nginx/extra/anti_scrapper.conf.template @@ -0,0 +1,19 @@ +# Rate limit error code (429) and definition (allow for bursts of 25 requests to load static images etc) and block immediately after the rate limit is applied to an IP +limit_req_status 429; +limit_req zone=scraperlimit burst=25 nodelay; + +location /robots.txt { + autoindex off; + alias /etc/nginx/templates/robots.txt; +} + +# Deny access to bots +# Block user agents that tend to be scrapers and badly behaved bots +if ($bad_bot = 1) { + return 444 "? beep boop ?"; +} + +# No scrapers +if ($scraper = 1) { + return 418 "?"; +} \ No newline at end of file diff --git a/nginx/extra/base_django.conf.template b/nginx/extra/base_django.conf.template new file mode 100644 index 000000000..28e1f4ed6 --- /dev/null +++ b/nginx/extra/base_django.conf.template @@ -0,0 +1,23 @@ +charset utf-8; +client_max_body_size 10480m; +client_body_buffer_size 32m; +sendfile on; +gzip on; + +location ~ /static/(.*) { + autoindex off; + access_log off; + include /etc/nginx/mime.types; + root /var/www/django/; +} +location ~ /media/(.*) { + autoindex off; + access_log off; + include /etc/nginx/mime.types; + root /var/www/django; +} +location /favicon.ico { + autoindex off; + access_log off; + alias /var/www/django/static/img/favicon.ico; +} \ No newline at end of file diff --git a/nginx/extra/block_map.conf.template b/nginx/extra/block_map.conf.template new file mode 100644 index 000000000..2ee634546 --- /dev/null +++ b/nginx/extra/block_map.conf.template @@ -0,0 +1,33 @@ +limit_req_zone $binary_remote_addr zone=scraperlimit:10m rate=${RATE_LIMIT}r/s; + +# Block bad bots +map $http_user_agent $bad_bot { + default 0; + ~*(?i)(JikeSpider) 1; + ~*(?i)(proximic) 1; + ~*(?i)(Sosospider) 1; + ~*(?i)(Baiduspider) 1; + ~*(?i)(Twitterbot) 1; + ~*(?i)(SemrushBot) 1; + ~*(?i)(^AIBOT) 1; + ~*(?i)(^BunnySlippers) 1; + ~*(?i)(^Cegbfeieh) 1; + ~*(?i)(^CheeseBot) 1; +} + +# Block scrappers +map $http_user_agent $scraper { + default 0; + ~*(?i)(Google-Extended) 1; + ~*(?i)(Applebot-Extended) 1; + ~*(?i)(anthropic-ai) 1; + ~*(?i)(ClaudeBot) 1; + ~*(?i)(Claude-Web) 1; + ~*(?i)(GPTBot) 1; + ~*(?i)(Omgili) 1; + ~*(?i)(FacebookBot) 1; + ~*(?i)(node-fetch) 1; + ~*(?i)(Timpibot) 1; + # If you don't provide a User-Agent, you can go away + ~*(^-$) 1; +} \ No newline at end of file diff --git a/nginx/extra/maintenance.conf.template b/nginx/extra/maintenance.conf.template new file mode 100644 index 000000000..8a71fd8ac --- /dev/null +++ b/nginx/extra/maintenance.conf.template @@ -0,0 +1,5 @@ +error_page 503 @maintenance; +location @maintenance { + root /srv; + try_files $uri /maintenance.html =503; +} \ No newline at end of file diff --git a/nginx/http/flower.conf.template b/nginx/http/flower.conf.template new file mode 100644 index 000000000..4db119655 --- /dev/null +++ b/nginx/http/flower.conf.template @@ -0,0 +1,7 @@ +server { + listen ${FLOWER_PUBLIC_PORT}; + location / { + proxy_pass http://flower:5555; + } + +} \ No newline at end of file diff --git a/nginx/http/minio.conf.template b/nginx/http/minio.conf.template new file mode 100644 index 000000000..451c8a87a --- /dev/null +++ b/nginx/http/minio.conf.template @@ -0,0 +1,32 @@ +server { + listen ${MINIO_PORT}; + # To allow special characters in headers + ignore_invalid_headers off; + # Allow any size file to be uploaded. + # Set to a value such as 1000m; to restrict file size to a specific value + client_max_body_size 0; + # To disable buffering + proxy_buffering off; + proxy_request_buffering off; + + location / { + # Default is HTTP/1, keepalive is only enabled in HTTP/1.1 + proxy_http_version 1.1; + proxy_set_header Connection ""; + chunked_transfer_encoding off; + + proxy_pass http://minio:9000; + } + + location /console/ { + rewrite ^/console/(.*)$ /$1 break; + + # To support websocket + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + + chunked_transfer_encoding off; + proxy_pass http://minio; + } +} \ No newline at end of file diff --git a/nginx/http/rabbit_console.conf.template b/nginx/http/rabbit_console.conf.template new file mode 100644 index 000000000..0d7d1623e --- /dev/null +++ b/nginx/http/rabbit_console.conf.template @@ -0,0 +1,6 @@ +server { + listen ${RABBITMQ_MANAGEMENT_PORT}; + location / { + proxy_pass http://rabbit:15672; + } +} \ No newline at end of file diff --git a/nginx/nginx.conf.template b/nginx/nginx.conf.template new file mode 100644 index 000000000..7d3b982a0 --- /dev/null +++ b/nginx/nginx.conf.template @@ -0,0 +1,113 @@ +user nginx; +load_module modules/ngx_http_acme_module.so; +worker_rlimit_nofile 65535; +worker_processes auto; +events { + worker_connections 4096; +} + +error_log /var/log/nginx/error.log warn; + +stream { + log_format proxy '$remote_addr [$time_local] ' + '$protocol $status $bytes_sent $bytes_received ' + '$session_time "$upstream_addr" ' + '"$upstream_bytes_sent" "$upstream_bytes_received" "$upstream_connect_time"'; + access_log /var/log/nginx/stream_access.log proxy buffer=32k flush=5s; + error_log /var/log/nginx/stream_error.log warn; + # Rabbit and Database connections + include stream/rabbit.conf; + include stream/database.conf; +} + +http { +access_log /var/log/nginx/access.log combined buffer=32k flush=5s; + +# Includes MinIO, Rabbit (console) +include http/*; +# Scrapper protection +include extra/block_map.conf; + +server_tokens off; + +# Necessary for ACME to work inside Docker +resolver 127.0.0.11 ipv6=off; +acme_issuer letsencrypt { + uri https://acme-v02.api.letsencrypt.org/directory; + contact ${EMAIL}; + state_path /var/cache/nginx/acme-letsencrypt; + + accept_terms_of_service; +} +acme_shared_zone zone=ngx_acme_shared:1M; + + +# Default server catchall for misconfigured machines trying to access the site +server { + server_name _; + listen *:80 default_server deferred; + return 444; +} + +# HTTP for the main instance +server { + set $force_https ${HTTPS}; + listen 80; + server_name ${DOMAIN_NAME}; + + include extra/base_django.conf; + include extra/anti_scrapper.conf; + location / { + if ($force_https ~* "true") { + return 301 https://$host$request_uri; + } + if (-f /srv/maintenance.on) { + return 503; + } + + proxy_intercept_errors on; + proxy_redirect off; + proxy_buffering off; + # To support websocket + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + + proxy_pass http://django:8000; + } + include extra/maintenance.conf; + +} + +# HTTPS for the main instance +server { + listen 443 ssl; + server_name ${DOMAIN_NAME}; + + acme_certificate letsencrypt; + ssl_certificate $acme_certificate; + ssl_certificate_key $acme_certificate_key; + # do not parse the certificate on each request + ssl_certificate_cache max=2; + + include extra/base_django.conf; + include extra/anti_scrapper.conf; + + location / { + if (-f /srv/maintenance.on) { + return 503; + } + + proxy_intercept_errors on; + proxy_redirect off; + proxy_buffering off; + # To support websocket + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + + proxy_pass http://django:8000; + } + include extra/maintenance.conf; +} +} \ No newline at end of file diff --git a/nginx/robots.txt b/nginx/robots.txt new file mode 100644 index 000000000..77470cb39 --- /dev/null +++ b/nginx/robots.txt @@ -0,0 +1,2 @@ +User-agent: * +Disallow: / \ No newline at end of file diff --git a/nginx/stream/database.conf.template b/nginx/stream/database.conf.template new file mode 100644 index 000000000..0dd391fe5 --- /dev/null +++ b/nginx/stream/database.conf.template @@ -0,0 +1,4 @@ +server { + listen 5432; + proxy_pass db:5432; +} \ No newline at end of file diff --git a/nginx/stream/rabbit.conf.template b/nginx/stream/rabbit.conf.template new file mode 100644 index 000000000..a606742f1 --- /dev/null +++ b/nginx/stream/rabbit.conf.template @@ -0,0 +1,4 @@ +server { + listen ${RABBITMQ_PORT}; + proxy_pass rabbit:5672; +} \ No newline at end of file diff --git a/tests/pytest.ini b/tests/pytest.ini index 1d3d9fa0c..ea06ae4f6 100644 --- a/tests/pytest.ini +++ b/tests/pytest.ini @@ -2,6 +2,6 @@ [pytest] # Use localhost as default host # addopts = --base-url=localhost --headed --browser webkit --browser firefox --browser chromium --numprocesses 2 -addopts = --base-url=http://localhost:8000 --browser firefox --screenshot only-on-failure --full-page-screenshot +addopts = --base-url=http://localhost --browser firefox --screenshot only-on-failure --full-page-screenshot log_cli = true log_cli_level = INFO diff --git a/tests/test_auth.py b/tests/test_auth.py index a1abe1c24..57248d2a7 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -9,7 +9,7 @@ def test_auth(browser: Browser) -> None: context = browser.new_context() page = context.new_page() - page.goto("http://localhost:8000") + page.goto("http://localhost") page.get_by_role("link", name="Login").click() page.get_by_role("textbox", name="username or email").click() page.get_by_role("textbox", name="username or email").fill( From d9280f38f38d71a357ab67814979620cecdfd0dd Mon Sep 17 00:00:00 2001 From: Obada Haddad Date: Tue, 3 Mar 2026 10:12:12 +0100 Subject: [PATCH 2/6] test to fix circle-ci failing test; add minio console address --- docker-compose.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 9c5f0f88b..37e26d9ac 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -70,13 +70,16 @@ services: #---------------------------------------------------------------------------------------------------- minio: image: minio/minio:RELEASE.2025-04-22T22-12-26Z - command: server /export + command: server /export --console-address ":9001" volumes: - ./var/minio:/export restart: unless-stopped ports: - 9000 + - 9001 env_file: .env + environment: + MINIO_BROWSER_REDIRECT_URL: "http://localhost/console" healthcheck: test: ["CMD", "curl", "-I", "http://minio:9000/minio/health/live"] interval: 5s @@ -272,4 +275,4 @@ services: networks: frontend: backend: - internal: true + #internal: true From adf54b0748f51d5aa5652d687faf3f815f471b23 Mon Sep 17 00:00:00 2001 From: Obada Haddad Date: Tue, 3 Mar 2026 13:05:59 +0100 Subject: [PATCH 3/6] more logs in circle-ci; remove buffering in nginx --- .circleci/config.yml | 2 ++ .env_circleci | 2 +- docker-compose.yml | 4 ++-- nginx/extra/base_django.conf.template | 5 +++-- nginx/http/flower.conf.template | 1 - nginx/http/minio.conf.template | 12 ------------ nginx/nginx.conf.template | 12 ++++++++++++ 7 files changed, 20 insertions(+), 18 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 208cb523d..0a5e3dc5b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -89,6 +89,8 @@ jobs: path: tests/test-results - store_artifacts: path: dockerLogs/ + - store_artifacts: + path: var/log/ workflows: version: 2 diff --git a/.env_circleci b/.env_circleci index d66c0b8e5..64315bf74 100644 --- a/.env_circleci +++ b/.env_circleci @@ -41,5 +41,5 @@ SUBMISSIONS_API_URL=http://django:8000/api # Nginx settings # ----------------------------------------------------------------------------- HTTPS=False -RATE_LIMIT=100 +RATE_LIMIT=10000 DOMAIN_NAME=localhost diff --git a/docker-compose.yml b/docker-compose.yml index 37e26d9ac..b7ab7558d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,7 @@ services: volumes: - ./src/staticfiles:/var/www/django/static - ./maintenance_mode/:/srv - - ./certsNginx:/var/cache/nginx/acme-letsencrypt + - ./nginx/certs/:/var/cache/nginx/acme-letsencrypt - ./nginx/:/etc/nginx/templates/ - ./var/log/nginx/:/var/log/nginx restart: unless-stopped @@ -275,4 +275,4 @@ services: networks: frontend: backend: - #internal: true + internal: true diff --git a/nginx/extra/base_django.conf.template b/nginx/extra/base_django.conf.template index 28e1f4ed6..31ffd6ab6 100644 --- a/nginx/extra/base_django.conf.template +++ b/nginx/extra/base_django.conf.template @@ -1,6 +1,7 @@ charset utf-8; -client_max_body_size 10480m; -client_body_buffer_size 32m; +client_max_body_size 0; +proxy_buffering off; +proxy_request_buffering off; sendfile on; gzip on; diff --git a/nginx/http/flower.conf.template b/nginx/http/flower.conf.template index 4db119655..574cc50bf 100644 --- a/nginx/http/flower.conf.template +++ b/nginx/http/flower.conf.template @@ -3,5 +3,4 @@ server { location / { proxy_pass http://flower:5555; } - } \ No newline at end of file diff --git a/nginx/http/minio.conf.template b/nginx/http/minio.conf.template index 451c8a87a..202557134 100644 --- a/nginx/http/minio.conf.template +++ b/nginx/http/minio.conf.template @@ -17,16 +17,4 @@ server { proxy_pass http://minio:9000; } - - location /console/ { - rewrite ^/console/(.*)$ /$1 break; - - # To support websocket - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; - - chunked_transfer_encoding off; - proxy_pass http://minio; - } } \ No newline at end of file diff --git a/nginx/nginx.conf.template b/nginx/nginx.conf.template index 7d3b982a0..373207cb7 100644 --- a/nginx/nginx.conf.template +++ b/nginx/nginx.conf.template @@ -75,6 +75,18 @@ server { proxy_pass http://django:8000; } + # MinIO console + location /console/ { + rewrite ^/console/(.*)$ /$1 break; + + # To support websocket + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + + chunked_transfer_encoding off; + proxy_pass http://minio:9001; + } include extra/maintenance.conf; } From 8680c98415d372bedb3fcb8b2f5e17ba34a270ba Mon Sep 17 00:00:00 2001 From: Obada Haddad Date: Tue, 3 Mar 2026 13:48:23 +0100 Subject: [PATCH 4/6] update gitignore and tentative fix on circle-ci tests --- .circleci/config.yml | 1 + .gitignore | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0a5e3dc5b..6652dd631 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -31,6 +31,7 @@ jobs: - run: name: "Setup: Create directories for MinIO (cannot be made by docker for some reason)" command: | + sudo echo "127.0.0.1 minio" >> /etc/hosts mkdir -p var/minio/public mkdir -p var/minio/private diff --git a/.gitignore b/.gitignore index ac9324edd..7f514db01 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,5 @@ caddy_data/ home_page_counters.json my-postgres.conf tests/config/state.json +state.json +certs/ From 62946b42158602664c685972c14f305049a27d71 Mon Sep 17 00:00:00 2001 From: Obada Haddad Date: Tue, 3 Mar 2026 13:59:21 +0100 Subject: [PATCH 5/6] tentative fix for nginx/minio circle-ci --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6652dd631..554fb5afd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -31,7 +31,7 @@ jobs: - run: name: "Setup: Create directories for MinIO (cannot be made by docker for some reason)" command: | - sudo echo "127.0.0.1 minio" >> /etc/hosts + echo "127.0.0.1 minio" | sudo tee -a /etc/hosts mkdir -p var/minio/public mkdir -p var/minio/private From b3b72c0c1b231eb5530754025af66d767ee69350 Mon Sep 17 00:00:00 2001 From: Obada Haddad Date: Thu, 5 Mar 2026 12:33:33 +0100 Subject: [PATCH 6/6] remove database access when not needed, put some volumes in read only in docker-compose.yml, make minio console work --- .env_circleci | 3 +- .env_sample | 56 ++++--- docker-compose.yml | 24 ++- nginx/extra/anti_scrapper.conf.template | 5 - nginx/http/minio.conf.template | 26 ++++ nginx/nginx.conf.template | 189 +++++++++++++----------- nginx/stream/database.conf.template | 2 +- 7 files changed, 172 insertions(+), 133 deletions(-) diff --git a/.env_circleci b/.env_circleci index 64315bf74..fb6f395c1 100644 --- a/.env_circleci +++ b/.env_circleci @@ -41,5 +41,6 @@ SUBMISSIONS_API_URL=http://django:8000/api # Nginx settings # ----------------------------------------------------------------------------- HTTPS=False -RATE_LIMIT=10000 +RATE_LIMIT=100 DOMAIN_NAME=localhost +DATABASE_ACCESS=True \ No newline at end of file diff --git a/.env_sample b/.env_sample index 854c8f9d3..3bee05258 100644 --- a/.env_sample +++ b/.env_sample @@ -1,17 +1,23 @@ -SECRET_KEY=change-this-secret +# Use openssl rand -hex 32 to generate this secret key, or generate it however you want and copy it here +SECRET_KEY= # For local setup and debug DEBUG=False +# ----------------------------------------------------------------------------- # Database +# ----------------------------------------------------------------------------- DB_HOST=db DB_NAME=postgres DB_USERNAME=postgres DB_PASSWORD=postgres DB_PORT=5432 +# ----------------------------------------------------------------------------- +# Django +# ----------------------------------------------------------------------------- DJANGO_SETTINGS_MODULE=settings.develop -ALLOWED_HOSTS=localhost,example.com +ALLOWED_HOSTS=localhost, SUBMISSIONS_API_URL=http://django:8000/api MAX_EXECUTION_TIME_LIMIT=600 # time limit for the default queue (in seconds) @@ -21,12 +27,11 @@ MAX_EXECUTION_TIME_LIMIT=600 # time limit for the default queue (in seconds) HTTPS=False RATE_LIMIT=5 DOMAIN_NAME=localhost - - -# SSL style domain definition TLS_EMAIL=your@email.com -# DOMAIN_NAME=example.com:443 +# ----------------------------------------------------------------------------- +# RabbitMQ +# ----------------------------------------------------------------------------- RABBITMQ_HOST=rabbit RABBITMQ_DEFAULT_USER=rabbit-username RABBITMQ_DEFAULT_PASS=rabbit-password-you-should-change @@ -34,11 +39,17 @@ RABBITMQ_MANAGEMENT_PORT=15672 RABBITMQ_PORT=5672 WORKER_CONNECTION_TIMEOUT=100000000 # milliseconds -FLOWER_PUBLIC_PORT=5555 +# ----------------------------------------------------------------------------- +# Flower +# ----------------------------------------------------------------------------- +FLOWER_PUBLIC_PORT=5555 FLOWER_BASIC_AUTH=root:password-you-should-change -SELENIUM_HOSTNAME=selenium + +# ----------------------------------------------------------------------------- +# Email Settings +# ----------------------------------------------------------------------------- # Uncomment to enable email settings #EMAIL_BACKEND=django.core.mail.backends.smtp.EmailBackend @@ -76,21 +87,6 @@ AWS_QUERYSTRING_AUTH=False #WORKER_BUNDLE_URL_REWRITE=http://localhost:9000|http://minio:9000 -# ----------------------------------------------------------------------------- -# Limit for re-running submission -# This is used to limit users to rerun submissions -# on default queue when number of submissions are < RERUN_SUBMISSION_LIMIT -# ----------------------------------------------------------------------------- -RERUN_SUBMISSION_LIMIT=30 - - -# ----------------------------------------------------------------------------- -# Enable or disbale regular email sign-in an sign-up -# ----------------------------------------------------------------------------- -ENABLE_SIGN_UP=True -ENABLE_SIGN_IN=True - - # # S3 storage example # STORAGE_TYPE=s3 # AWS_ACCESS_KEY_ID=12312312312312312331223 @@ -113,6 +109,20 @@ ENABLE_SIGN_IN=True # GS_PRIVATE_BUCKET_NAME=private # GOOGLE_APPLICATION_CREDENTIALS=/app/certs/google-storage-api.json +# ----------------------------------------------------------------------------- +# Limit for re-running submission +# This is used to limit users to rerun submissions +# on default queue when number of submissions are < RERUN_SUBMISSION_LIMIT +# ----------------------------------------------------------------------------- +RERUN_SUBMISSION_LIMIT=30 + + +# ----------------------------------------------------------------------------- +# Enable or disbale regular email sign-in an sign-up +# ----------------------------------------------------------------------------- +ENABLE_SIGN_UP=True +ENABLE_SIGN_IN=True + # ----------------------------------------------------------------------------- # Logging (Serialized outputs the logs in JSON format) diff --git a/docker-compose.yml b/docker-compose.yml index b7ab7558d..d17cb45ce 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,16 +3,16 @@ services: # Web Services #---------------------------------------------------------------------------------------------------- nginx: - image: nginx:latest + image: nginx:alpine env_file: .env environment: - NGINX_ENVSUBST_OUTPUT_DIR=/etc/nginx command: ["nginx", "-g", "daemon off;"] volumes: - - ./src/staticfiles:/var/www/django/static - - ./maintenance_mode/:/srv + - ./src/staticfiles:/var/www/django/static:ro + - ./maintenance_mode/:/srv:ro + - ./nginx/:/etc/nginx/templates/:ro - ./nginx/certs/:/var/cache/nginx/acme-letsencrypt - - ./nginx/:/etc/nginx/templates/ - ./var/log/nginx/:/var/log/nginx restart: unless-stopped ports: @@ -22,14 +22,8 @@ services: - 5432:5432 - ${RABBITMQ_MANAGEMENT_PORT:-15672}:15672 - ${RABBITMQ_PORT}:5672 - - ${MINIO_PORT}:9000 + - ${MINIO_PORT:-9000}:9000 - ${FLOWER_PUBLIC_PORT:-5555}:5555 - depends_on: - - django - - minio - - rabbit - - site_worker - - db networks: - frontend - backend @@ -63,6 +57,7 @@ services: max-file: "5" networks: - backend + - frontend #---------------------------------------------------------------------------------------------------- @@ -79,7 +74,7 @@ services: - 9001 env_file: .env environment: - MINIO_BROWSER_REDIRECT_URL: "http://localhost/console" + MINIO_BROWSER_REDIRECT_URL: "http://${DOMAIN_NAME}/console" healthcheck: test: ["CMD", "curl", "-I", "http://minio:9000/minio/health/live"] interval: 5s @@ -146,7 +141,7 @@ services: volumes: - ./var/postgres:/var/lib/postgresql/18/:delegated - ./backups:/app/backups - - ./my-postgres.conf:/etc/postgresql/postgresql.conf + - ./my-postgres.conf:/etc/postgresql/postgresql.conf:ro restart: unless-stopped logging: options: @@ -253,7 +248,7 @@ services: - django - rabbit volumes: - - ./compute_worker:/app + - ./compute_worker:/app:ro - ${HOST_DIRECTORY:-/tmp/codabench}:/codabench # Actual connection back to docker parent to run things - /var/run/docker.sock:/var/run/docker.sock @@ -271,6 +266,7 @@ services: max-file: "5" networks: - backend + - frontend networks: frontend: diff --git a/nginx/extra/anti_scrapper.conf.template b/nginx/extra/anti_scrapper.conf.template index d60c26751..607beaa0d 100644 --- a/nginx/extra/anti_scrapper.conf.template +++ b/nginx/extra/anti_scrapper.conf.template @@ -2,11 +2,6 @@ limit_req_status 429; limit_req zone=scraperlimit burst=25 nodelay; -location /robots.txt { - autoindex off; - alias /etc/nginx/templates/robots.txt; -} - # Deny access to bots # Block user agents that tend to be scrapers and badly behaved bots if ($bad_bot = 1) { diff --git a/nginx/http/minio.conf.template b/nginx/http/minio.conf.template index 202557134..fee0447ce 100644 --- a/nginx/http/minio.conf.template +++ b/nginx/http/minio.conf.template @@ -10,6 +10,8 @@ server { proxy_request_buffering off; location / { + include extra/anti_scrapper.conf; + # Default is HTTP/1, keepalive is only enabled in HTTP/1.1 proxy_http_version 1.1; proxy_set_header Connection ""; @@ -17,4 +19,28 @@ server { proxy_pass http://minio:9000; } + + # MinIO console + location /console/ { + include extra/anti_scrapper.conf; + + rewrite ^/console/(.*)$ /$1 break; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-NginX-Proxy true; + + # This is necessary to pass the correct IP to be hashed + real_ip_header X-Real-IP; + proxy_connect_timeout 300; + + # To support websocket + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + + chunked_transfer_encoding off; + proxy_pass http://minio:9001; + } } \ No newline at end of file diff --git a/nginx/nginx.conf.template b/nginx/nginx.conf.template index 373207cb7..a95df7e55 100644 --- a/nginx/nginx.conf.template +++ b/nginx/nginx.conf.template @@ -9,117 +9,128 @@ events { error_log /var/log/nginx/error.log warn; stream { + resolver 127.0.0.11 ipv6=off; log_format proxy '$remote_addr [$time_local] ' - '$protocol $status $bytes_sent $bytes_received ' - '$session_time "$upstream_addr" ' - '"$upstream_bytes_sent" "$upstream_bytes_received" "$upstream_connect_time"'; + '$protocol $status $bytes_sent $bytes_received ' + '$session_time "$upstream_addr" ' + '"$upstream_bytes_sent" "$upstream_bytes_received" "$upstream_connect_time"'; access_log /var/log/nginx/stream_access.log proxy buffer=32k flush=5s; error_log /var/log/nginx/stream_error.log warn; - # Rabbit and Database connections + + map "" $database_access { + default ${DATABASE_ACCESS}; + } + map $database_access $backend { + default no_access; + "True" db:5432; + } + # Rabbit and Database include stream/rabbit.conf; include stream/database.conf; } http { -access_log /var/log/nginx/access.log combined buffer=32k flush=5s; + access_log /var/log/nginx/access.log combined buffer=32k flush=5s; -# Includes MinIO, Rabbit (console) -include http/*; -# Scrapper protection -include extra/block_map.conf; + # Scrapper protection + include extra/block_map.conf; -server_tokens off; + # Includes MinIO, Rabbit (console) + include http/flower.conf; + include http/minio.conf; + include http/rabbit_console.conf; -# Necessary for ACME to work inside Docker -resolver 127.0.0.11 ipv6=off; -acme_issuer letsencrypt { - uri https://acme-v02.api.letsencrypt.org/directory; - contact ${EMAIL}; - state_path /var/cache/nginx/acme-letsencrypt; + server_tokens off; - accept_terms_of_service; -} -acme_shared_zone zone=ngx_acme_shared:1M; + # Necessary for ACME to work inside Docker + resolver 127.0.0.11 ipv6=off; + acme_issuer letsencrypt { + uri https://acme-v02.api.letsencrypt.org/directory; + contact ${EMAIL}; + state_path /var/cache/nginx/acme-letsencrypt; + accept_terms_of_service; + } + acme_shared_zone zone=ngx_acme_shared:1M; -# Default server catchall for misconfigured machines trying to access the site -server { - server_name _; - listen *:80 default_server deferred; - return 444; -} -# HTTP for the main instance -server { - set $force_https ${HTTPS}; - listen 80; - server_name ${DOMAIN_NAME}; - - include extra/base_django.conf; - include extra/anti_scrapper.conf; - location / { - if ($force_https ~* "true") { - return 301 https://$host$request_uri; + # Default server catchall for misconfigured machines trying to access the site + server { + server_name _; + listen *:80 default_server deferred; + return 444; + } + + # HTTP for the main instance + server { + set $force_https ${HTTPS}; + listen 80; + server_name ${DOMAIN_NAME}; + + include extra/base_django.conf; + include extra/anti_scrapper.conf; + + location /robots.txt { + autoindex off; + alias /etc/nginx/templates/robots.txt; } - if (-f /srv/maintenance.on) { - return 503; + location / { + if ($force_https ~* "true") { + return 301 https://$host$request_uri; + } + if (-f /srv/maintenance.on) { + return 503; + } + + proxy_intercept_errors on; + proxy_redirect off; + proxy_buffering off; + # To support websocket + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + + proxy_pass http://django:8000; } - - proxy_intercept_errors on; - proxy_redirect off; - proxy_buffering off; - # To support websocket - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; - - proxy_pass http://django:8000; + # MinIO console + include extra/maintenance.conf; + } - # MinIO console - location /console/ { - rewrite ^/console/(.*)$ /$1 break; - # To support websocket - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; + # HTTPS for the main instance + server { + listen 443 ssl; + server_name ${DOMAIN_NAME}; - chunked_transfer_encoding off; - proxy_pass http://minio:9001; - } - include extra/maintenance.conf; + acme_certificate letsencrypt; + ssl_certificate $acme_certificate; + ssl_certificate_key $acme_certificate_key; + # do not parse the certificate on each request + ssl_certificate_cache max=2; -} + include extra/base_django.conf; + include extra/anti_scrapper.conf; -# HTTPS for the main instance -server { - listen 443 ssl; - server_name ${DOMAIN_NAME}; - - acme_certificate letsencrypt; - ssl_certificate $acme_certificate; - ssl_certificate_key $acme_certificate_key; - # do not parse the certificate on each request - ssl_certificate_cache max=2; - - include extra/base_django.conf; - include extra/anti_scrapper.conf; - - location / { - if (-f /srv/maintenance.on) { - return 503; + location /robots.txt { + autoindex off; + alias /etc/nginx/templates/robots.txt; } - - proxy_intercept_errors on; - proxy_redirect off; - proxy_buffering off; - # To support websocket - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; - - proxy_pass http://django:8000; + + location / { + if (-f /srv/maintenance.on) { + return 503; + } + + proxy_intercept_errors on; + proxy_redirect off; + proxy_buffering off; + # To support websocket + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + + proxy_pass http://django:8000; + } + include extra/maintenance.conf; } - include extra/maintenance.conf; -} } \ No newline at end of file diff --git a/nginx/stream/database.conf.template b/nginx/stream/database.conf.template index 0dd391fe5..86dd01c8e 100644 --- a/nginx/stream/database.conf.template +++ b/nginx/stream/database.conf.template @@ -1,4 +1,4 @@ server { listen 5432; - proxy_pass db:5432; + proxy_pass $backend; } \ No newline at end of file