diff --git a/.circleci/config.yml b/.circleci/config.yml index 208cb523d..554fb5afd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -31,6 +31,7 @@ jobs: - run: name: "Setup: Create directories for MinIO (cannot be made by docker for some reason)" command: | + echo "127.0.0.1 minio" | sudo tee -a /etc/hosts mkdir -p var/minio/public mkdir -p var/minio/private @@ -89,6 +90,8 @@ jobs: path: tests/test-results - store_artifacts: path: dockerLogs/ + - store_artifacts: + path: var/log/ workflows: version: 2 diff --git a/.env_circleci b/.env_circleci index f56adf935..fb6f395c1 100644 --- a/.env_circleci +++ b/.env_circleci @@ -8,12 +8,13 @@ DB_PORT=5432 RABBITMQ_DEFAULT_USER=rabbit-username RABBITMQ_DEFAULT_PASS=rabbit-password-you-should-change +RABBITMQ_MANAGEMENT_PORT=15672 RABBITMQ_PORT=5672 RABBITMQ_HOST=rabbit WORKER_CONNECTION_TIMEOUT=100000000 # milliseconds FLOWER_BASIC_AUTH=root:password-you-should-change - +FLOWER_PUBLIC_PORT=5555 DJANGO_SETTINGS_MODULE=settings.test # Minio local storage example @@ -28,11 +29,18 @@ AWS_SECRET_ACCESS_KEY=testsecret AWS_STORAGE_BUCKET_NAME=public AWS_STORAGE_PRIVATE_BUCKET_NAME=private # NOTE! port 9000 here should match $MINIO_PORT -AWS_S3_ENDPOINT_URL=http://172.17.0.1:9000/ +AWS_S3_ENDPOINT_URL=http://minio:9000/ AWS_QUERYSTRING_AUTH=False DJANGO_SUPERUSER_PASSWORD=codabench DJANGO_SUPERUSER_EMAIL=test@test.com DJANGO_SUPERUSER_USERNAME=codabench -DOMAIN_NAME=localhost:80 TLS_EMAIL=your@email.com -SUBMISSIONS_API_URL=http://django:8000/api \ No newline at end of file +SUBMISSIONS_API_URL=http://django:8000/api + +# ----------------------------------------------------------------------------- +# Nginx settings +# ----------------------------------------------------------------------------- +HTTPS=False +RATE_LIMIT=100 +DOMAIN_NAME=localhost +DATABASE_ACCESS=True \ No newline at end of file diff --git a/.env_sample b/.env_sample index 6bd01cfbd..3bee05258 100644 --- a/.env_sample +++ b/.env_sample @@ -1,42 +1,55 @@ -SECRET_KEY=change-this-secret +# Use openssl rand -hex 32 to generate this secret key, or generate it however you want and copy it here +SECRET_KEY= # For local setup and debug -DEBUG=True +DEBUG=False +# ----------------------------------------------------------------------------- # Database +# ----------------------------------------------------------------------------- DB_HOST=db DB_NAME=postgres DB_USERNAME=postgres DB_PASSWORD=postgres DB_PORT=5432 +# ----------------------------------------------------------------------------- +# Django +# ----------------------------------------------------------------------------- DJANGO_SETTINGS_MODULE=settings.develop -ALLOWED_HOSTS=localhost,example.com +ALLOWED_HOSTS=localhost, SUBMISSIONS_API_URL=http://django:8000/api MAX_EXECUTION_TIME_LIMIT=600 # time limit for the default queue (in seconds) -# Local domain definition -DOMAIN_NAME=localhost:80 - -# SSL style domain definition +# ----------------------------------------------------------------------------- +# Nginx settings +# ----------------------------------------------------------------------------- +HTTPS=False +RATE_LIMIT=5 +DOMAIN_NAME=localhost TLS_EMAIL=your@email.com -# DOMAIN_NAME=example.com:443 +# ----------------------------------------------------------------------------- +# RabbitMQ +# ----------------------------------------------------------------------------- RABBITMQ_HOST=rabbit RABBITMQ_DEFAULT_USER=rabbit-username RABBITMQ_DEFAULT_PASS=rabbit-password-you-should-change RABBITMQ_MANAGEMENT_PORT=15672 RABBITMQ_PORT=5672 WORKER_CONNECTION_TIMEOUT=100000000 # milliseconds -#RABBITMQ_HTTP_PROXY=http://proxy-example:3128 -#RABBITMQ_HTTPS_PROXY=http://proxy-example:3128 -#RABBITMQ_NO_PROXY=localhost,172.0.0.0/8 -FLOWER_PUBLIC_PORT=5555 +# ----------------------------------------------------------------------------- +# Flower +# ----------------------------------------------------------------------------- +FLOWER_PUBLIC_PORT=5555 FLOWER_BASIC_AUTH=root:password-you-should-change -SELENIUM_HOSTNAME=selenium + +# ----------------------------------------------------------------------------- +# Email Settings +# ----------------------------------------------------------------------------- # Uncomment to enable email settings #EMAIL_BACKEND=django.core.mail.backends.smtp.EmailBackend @@ -74,21 +87,6 @@ AWS_QUERYSTRING_AUTH=False #WORKER_BUNDLE_URL_REWRITE=http://localhost:9000|http://minio:9000 -# ----------------------------------------------------------------------------- -# Limit for re-running submission -# This is used to limit users to rerun submissions -# on default queue when number of submissions are < RERUN_SUBMISSION_LIMIT -# ----------------------------------------------------------------------------- -RERUN_SUBMISSION_LIMIT=30 - - -# ----------------------------------------------------------------------------- -# Enable or disbale regular email sign-in an sign-up -# ----------------------------------------------------------------------------- -ENABLE_SIGN_UP=True -ENABLE_SIGN_IN=True - - # # S3 storage example # STORAGE_TYPE=s3 # AWS_ACCESS_KEY_ID=12312312312312312331223 @@ -111,6 +109,20 @@ ENABLE_SIGN_IN=True # GS_PRIVATE_BUCKET_NAME=private # GOOGLE_APPLICATION_CREDENTIALS=/app/certs/google-storage-api.json +# ----------------------------------------------------------------------------- +# Limit for re-running submission +# This is used to limit users to rerun submissions +# on default queue when number of submissions are < RERUN_SUBMISSION_LIMIT +# ----------------------------------------------------------------------------- +RERUN_SUBMISSION_LIMIT=30 + + +# ----------------------------------------------------------------------------- +# Enable or disbale regular email sign-in an sign-up +# ----------------------------------------------------------------------------- +ENABLE_SIGN_UP=True +ENABLE_SIGN_IN=True + # ----------------------------------------------------------------------------- # Logging (Serialized outputs the logs in JSON format) diff --git a/.gitignore b/.gitignore index ac9324edd..7f514db01 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,5 @@ caddy_data/ home_page_counters.json my-postgres.conf tests/config/state.json +state.json +certs/ diff --git a/docker-compose.yml b/docker-compose.yml index 46c1783de..d17cb45ce 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,24 +2,31 @@ services: #---------------------------------------------------------------------------------------------------- # Web Services #---------------------------------------------------------------------------------------------------- - caddy: - image: caddy:2.10.0 + nginx: + image: nginx:alpine env_file: .env environment: - - ACME_AGREE=true + - NGINX_ENVSUBST_OUTPUT_DIR=/etc/nginx + command: ["nginx", "-g", "daemon off;"] volumes: - - ./Caddyfile:/etc/caddy/Caddyfile - - ./src/staticfiles:/var/www/django/static - - ./caddy_data:/data - - ./caddy_config:/config - - ./var/log/caddy:/var/log/ - - ./maintenance_mode/:/srv + - ./src/staticfiles:/var/www/django/static:ro + - ./maintenance_mode/:/srv:ro + - ./nginx/:/etc/nginx/templates/:ro + - ./nginx/certs/:/var/cache/nginx/acme-letsencrypt + - ./var/log/nginx/:/var/log/nginx restart: unless-stopped ports: - 80:80 - 443:443 - depends_on: - - django + - 8000:8000 + - 5432:5432 + - ${RABBITMQ_MANAGEMENT_PORT:-15672}:15672 + - ${RABBITMQ_PORT}:5672 + - ${MINIO_PORT:-9000}:9000 + - ${FLOWER_PUBLIC_PORT:-5555}:5555 + networks: + - frontend + - backend django: build: @@ -37,7 +44,7 @@ services: - ./var/logs:/app/logs restart: unless-stopped ports: - - 8000:8000 + - 8000 depends_on: - db - rabbit @@ -48,6 +55,9 @@ services: options: max-size: "20m" max-file: "5" + networks: + - backend + - frontend #---------------------------------------------------------------------------------------------------- @@ -55,17 +65,22 @@ services: #---------------------------------------------------------------------------------------------------- minio: image: minio/minio:RELEASE.2025-04-22T22-12-26Z - command: server /export + command: server /export --console-address ":9001" volumes: - ./var/minio:/export restart: unless-stopped ports: - - $MINIO_PORT:9000 + - 9000 + - 9001 env_file: .env + environment: + MINIO_BROWSER_REDIRECT_URL: "http://${DOMAIN_NAME}/console" healthcheck: test: ["CMD", "curl", "-I", "http://minio:9000/minio/health/live"] interval: 5s retries: 5 + networks: + - backend createbuckets: image: minio/mc:RELEASE.2025-07-21T05-28-08Z depends_on: @@ -90,6 +105,8 @@ services: fi; exit 0; " + networks: + - backend #---------------------------------------------------------------------------------------------------- # Local development helper, rebuilds RiotJS/Stylus on change @@ -106,7 +123,6 @@ services: max-size: "20m" max-file: "5" - #---------------------------------------------------------------------------------------------------- # Database Service # @@ -121,16 +137,18 @@ services: - POSTGRES_PASSWORD=${DB_PASSWORD} command: ["postgres", "-c", "log_statement=all", "-c", "log_destination=stderr", "-c", "config_file=/etc/postgresql/postgresql.conf"] ports: - - 5432:5432 + - 5432 volumes: - ./var/postgres:/var/lib/postgresql/18/:delegated - ./backups:/app/backups - - ./my-postgres.conf:/etc/postgresql/postgresql.conf + - ./my-postgres.conf:/etc/postgresql/postgresql.conf:ro restart: unless-stopped logging: options: max-size: "20m" max-file: "5" + networks: + - backend #---------------------------------------------------------------------------------------------------- # Rabbitmq & Flower monitoring tool @@ -145,13 +163,9 @@ services: # containers being destroyed..! hostname: rabbit env_file: .env - environment: - - http_proxy=${RABBITMQ_HTTP_PROXY} - - https_proxy=${RABBITMQ_HTTPS_PROXY} - - no_proxy=${RABBITMQ_NO_PROXY} ports: - - ${RABBITMQ_MANAGEMENT_PORT:-15672}:15672 - - ${RABBITMQ_PORT}:5672 + - 15672 + - 5672 volumes: - ./var/rabbit:/var/lib/rabbitmq restart: unless-stopped @@ -159,6 +173,8 @@ services: options: max-size: "20m" max-file: "5" + networks: + - backend flower: image: mher/flower @@ -167,13 +183,15 @@ services: - CELERY_BROKER_URL=pyamqp://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@${RABBITMQ_HOST}:${RABBITMQ_PORT}// restart: unless-stopped ports: - - ${FLOWER_PUBLIC_PORT:-5555}:5555 + - 5555 depends_on: - rabbit logging: options: max-size: "20m" max-file: "5" + networks: + - backend #---------------------------------------------------------------------------------------------------- # Redis @@ -181,12 +199,14 @@ services: redis: image: redis ports: - - 6379:6379 + - 6379 restart: unless-stopped logging: options: max-size: "20m" max-file: "5" + networks: + - backend #---------------------------------------------------------------------------------------------------- # Celery Service @@ -215,6 +235,8 @@ services: # Limit memory substantially here so we see any problems that may # appear on Heroku ahead of time memory: 256M + networks: + - backend compute_worker: command: ["celery -A compute_worker worker -l info -Q compute-worker -n compute-worker@%n"] @@ -226,7 +248,7 @@ services: - django - rabbit volumes: - - ./compute_worker:/app + - ./compute_worker:/app:ro - ${HOST_DIRECTORY:-/tmp/codabench}:/codabench # Actual connection back to docker parent to run things - /var/run/docker.sock:/var/run/docker.sock @@ -242,3 +264,11 @@ services: options: max-size: "20m" max-file: "5" + networks: + - backend + - frontend + +networks: + frontend: + backend: + internal: true diff --git a/nginx/extra/anti_scrapper.conf.template b/nginx/extra/anti_scrapper.conf.template new file mode 100644 index 000000000..607beaa0d --- /dev/null +++ b/nginx/extra/anti_scrapper.conf.template @@ -0,0 +1,14 @@ +# Rate limit error code (429) and definition (allow for bursts of 25 requests to load static images etc) and block immediately after the rate limit is applied to an IP +limit_req_status 429; +limit_req zone=scraperlimit burst=25 nodelay; + +# Deny access to bots +# Block user agents that tend to be scrapers and badly behaved bots +if ($bad_bot = 1) { + return 444 "? beep boop ?"; +} + +# No scrapers +if ($scraper = 1) { + return 418 "?"; +} \ No newline at end of file diff --git a/nginx/extra/base_django.conf.template b/nginx/extra/base_django.conf.template new file mode 100644 index 000000000..31ffd6ab6 --- /dev/null +++ b/nginx/extra/base_django.conf.template @@ -0,0 +1,24 @@ +charset utf-8; +client_max_body_size 0; +proxy_buffering off; +proxy_request_buffering off; +sendfile on; +gzip on; + +location ~ /static/(.*) { + autoindex off; + access_log off; + include /etc/nginx/mime.types; + root /var/www/django/; +} +location ~ /media/(.*) { + autoindex off; + access_log off; + include /etc/nginx/mime.types; + root /var/www/django; +} +location /favicon.ico { + autoindex off; + access_log off; + alias /var/www/django/static/img/favicon.ico; +} \ No newline at end of file diff --git a/nginx/extra/block_map.conf.template b/nginx/extra/block_map.conf.template new file mode 100644 index 000000000..2ee634546 --- /dev/null +++ b/nginx/extra/block_map.conf.template @@ -0,0 +1,33 @@ +limit_req_zone $binary_remote_addr zone=scraperlimit:10m rate=${RATE_LIMIT}r/s; + +# Block bad bots +map $http_user_agent $bad_bot { + default 0; + ~*(?i)(JikeSpider) 1; + ~*(?i)(proximic) 1; + ~*(?i)(Sosospider) 1; + ~*(?i)(Baiduspider) 1; + ~*(?i)(Twitterbot) 1; + ~*(?i)(SemrushBot) 1; + ~*(?i)(^AIBOT) 1; + ~*(?i)(^BunnySlippers) 1; + ~*(?i)(^Cegbfeieh) 1; + ~*(?i)(^CheeseBot) 1; +} + +# Block scrappers +map $http_user_agent $scraper { + default 0; + ~*(?i)(Google-Extended) 1; + ~*(?i)(Applebot-Extended) 1; + ~*(?i)(anthropic-ai) 1; + ~*(?i)(ClaudeBot) 1; + ~*(?i)(Claude-Web) 1; + ~*(?i)(GPTBot) 1; + ~*(?i)(Omgili) 1; + ~*(?i)(FacebookBot) 1; + ~*(?i)(node-fetch) 1; + ~*(?i)(Timpibot) 1; + # If you don't provide a User-Agent, you can go away + ~*(^-$) 1; +} \ No newline at end of file diff --git a/nginx/extra/maintenance.conf.template b/nginx/extra/maintenance.conf.template new file mode 100644 index 000000000..8a71fd8ac --- /dev/null +++ b/nginx/extra/maintenance.conf.template @@ -0,0 +1,5 @@ +error_page 503 @maintenance; +location @maintenance { + root /srv; + try_files $uri /maintenance.html =503; +} \ No newline at end of file diff --git a/nginx/http/flower.conf.template b/nginx/http/flower.conf.template new file mode 100644 index 000000000..574cc50bf --- /dev/null +++ b/nginx/http/flower.conf.template @@ -0,0 +1,6 @@ +server { + listen ${FLOWER_PUBLIC_PORT}; + location / { + proxy_pass http://flower:5555; + } +} \ No newline at end of file diff --git a/nginx/http/minio.conf.template b/nginx/http/minio.conf.template new file mode 100644 index 000000000..fee0447ce --- /dev/null +++ b/nginx/http/minio.conf.template @@ -0,0 +1,46 @@ +server { + listen ${MINIO_PORT}; + # To allow special characters in headers + ignore_invalid_headers off; + # Allow any size file to be uploaded. + # Set to a value such as 1000m; to restrict file size to a specific value + client_max_body_size 0; + # To disable buffering + proxy_buffering off; + proxy_request_buffering off; + + location / { + include extra/anti_scrapper.conf; + + # Default is HTTP/1, keepalive is only enabled in HTTP/1.1 + proxy_http_version 1.1; + proxy_set_header Connection ""; + chunked_transfer_encoding off; + + proxy_pass http://minio:9000; + } + + # MinIO console + location /console/ { + include extra/anti_scrapper.conf; + + rewrite ^/console/(.*)$ /$1 break; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-NginX-Proxy true; + + # This is necessary to pass the correct IP to be hashed + real_ip_header X-Real-IP; + proxy_connect_timeout 300; + + # To support websocket + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + + chunked_transfer_encoding off; + proxy_pass http://minio:9001; + } +} \ No newline at end of file diff --git a/nginx/http/rabbit_console.conf.template b/nginx/http/rabbit_console.conf.template new file mode 100644 index 000000000..0d7d1623e --- /dev/null +++ b/nginx/http/rabbit_console.conf.template @@ -0,0 +1,6 @@ +server { + listen ${RABBITMQ_MANAGEMENT_PORT}; + location / { + proxy_pass http://rabbit:15672; + } +} \ No newline at end of file diff --git a/nginx/nginx.conf.template b/nginx/nginx.conf.template new file mode 100644 index 000000000..a95df7e55 --- /dev/null +++ b/nginx/nginx.conf.template @@ -0,0 +1,136 @@ +user nginx; +load_module modules/ngx_http_acme_module.so; +worker_rlimit_nofile 65535; +worker_processes auto; +events { + worker_connections 4096; +} + +error_log /var/log/nginx/error.log warn; + +stream { + resolver 127.0.0.11 ipv6=off; + log_format proxy '$remote_addr [$time_local] ' + '$protocol $status $bytes_sent $bytes_received ' + '$session_time "$upstream_addr" ' + '"$upstream_bytes_sent" "$upstream_bytes_received" "$upstream_connect_time"'; + access_log /var/log/nginx/stream_access.log proxy buffer=32k flush=5s; + error_log /var/log/nginx/stream_error.log warn; + + map "" $database_access { + default ${DATABASE_ACCESS}; + } + map $database_access $backend { + default no_access; + "True" db:5432; + } + # Rabbit and Database + include stream/rabbit.conf; + include stream/database.conf; +} + +http { + access_log /var/log/nginx/access.log combined buffer=32k flush=5s; + + # Scrapper protection + include extra/block_map.conf; + + # Includes MinIO, Rabbit (console) + include http/flower.conf; + include http/minio.conf; + include http/rabbit_console.conf; + + server_tokens off; + + # Necessary for ACME to work inside Docker + resolver 127.0.0.11 ipv6=off; + acme_issuer letsencrypt { + uri https://acme-v02.api.letsencrypt.org/directory; + contact ${EMAIL}; + state_path /var/cache/nginx/acme-letsencrypt; + + accept_terms_of_service; + } + acme_shared_zone zone=ngx_acme_shared:1M; + + + # Default server catchall for misconfigured machines trying to access the site + server { + server_name _; + listen *:80 default_server deferred; + return 444; + } + + # HTTP for the main instance + server { + set $force_https ${HTTPS}; + listen 80; + server_name ${DOMAIN_NAME}; + + include extra/base_django.conf; + include extra/anti_scrapper.conf; + + location /robots.txt { + autoindex off; + alias /etc/nginx/templates/robots.txt; + } + location / { + if ($force_https ~* "true") { + return 301 https://$host$request_uri; + } + if (-f /srv/maintenance.on) { + return 503; + } + + proxy_intercept_errors on; + proxy_redirect off; + proxy_buffering off; + # To support websocket + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + + proxy_pass http://django:8000; + } + # MinIO console + include extra/maintenance.conf; + + } + + # HTTPS for the main instance + server { + listen 443 ssl; + server_name ${DOMAIN_NAME}; + + acme_certificate letsencrypt; + ssl_certificate $acme_certificate; + ssl_certificate_key $acme_certificate_key; + # do not parse the certificate on each request + ssl_certificate_cache max=2; + + include extra/base_django.conf; + include extra/anti_scrapper.conf; + + location /robots.txt { + autoindex off; + alias /etc/nginx/templates/robots.txt; + } + + location / { + if (-f /srv/maintenance.on) { + return 503; + } + + proxy_intercept_errors on; + proxy_redirect off; + proxy_buffering off; + # To support websocket + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + + proxy_pass http://django:8000; + } + include extra/maintenance.conf; + } +} \ No newline at end of file diff --git a/nginx/robots.txt b/nginx/robots.txt new file mode 100644 index 000000000..77470cb39 --- /dev/null +++ b/nginx/robots.txt @@ -0,0 +1,2 @@ +User-agent: * +Disallow: / \ No newline at end of file diff --git a/nginx/stream/database.conf.template b/nginx/stream/database.conf.template new file mode 100644 index 000000000..86dd01c8e --- /dev/null +++ b/nginx/stream/database.conf.template @@ -0,0 +1,4 @@ +server { + listen 5432; + proxy_pass $backend; +} \ No newline at end of file diff --git a/nginx/stream/rabbit.conf.template b/nginx/stream/rabbit.conf.template new file mode 100644 index 000000000..a606742f1 --- /dev/null +++ b/nginx/stream/rabbit.conf.template @@ -0,0 +1,4 @@ +server { + listen ${RABBITMQ_PORT}; + proxy_pass rabbit:5672; +} \ No newline at end of file diff --git a/tests/pytest.ini b/tests/pytest.ini index 1d3d9fa0c..ea06ae4f6 100644 --- a/tests/pytest.ini +++ b/tests/pytest.ini @@ -2,6 +2,6 @@ [pytest] # Use localhost as default host # addopts = --base-url=localhost --headed --browser webkit --browser firefox --browser chromium --numprocesses 2 -addopts = --base-url=http://localhost:8000 --browser firefox --screenshot only-on-failure --full-page-screenshot +addopts = --base-url=http://localhost --browser firefox --screenshot only-on-failure --full-page-screenshot log_cli = true log_cli_level = INFO diff --git a/tests/test_auth.py b/tests/test_auth.py index a1abe1c24..57248d2a7 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -9,7 +9,7 @@ def test_auth(browser: Browser) -> None: context = browser.new_context() page = context.new_page() - page.goto("http://localhost:8000") + page.goto("http://localhost") page.get_by_role("link", name="Login").click() page.get_by_role("textbox", name="username or email").click() page.get_by_role("textbox", name="username or email").fill(