docker-compose.yml

version: '3.7'

services:

  # Fluentd Microservice
  # This is the `fluentd` logger container that ingests all logs from Docker and
  # acts as a centralised logging layer.
  fluentd:
    build:
      context: ./services/fluentd
      dockerfile: Dockerfile
      labels:
        arclytics.io: "development"
    image: neuraldev/arc_fluentd:1.0
    restart: always
    ports:
      - 24224:24224
      - 24224:24224/udp
      - 9880:9880
    expose:
      - 24224
      - 9880
    volumes:
      - ./services/fluentd/logs:/fluentd/log
    depends_on:
      - elasticsearch
    networks:
      - dev_subnet

  # Client Microservice
  # This is the React front-end container that the client can access via the
  # Nginx HTTP server. This container is run in development mode as it is only
  # being used in the development environment.
  client:
    build:
      context: ./services/client
      dockerfile: Dockerfile
      labels:
        arclytics.io: "development"
    image: neuraldev/arc_sim_client:1.2
    volumes:
      - ./services/client:/usr/src/app
      - /usr/src/app/node_modules
    logging:
      driver: fluentd
      options:
        fluentd-address: localhost:24224
        tag: client.docker.access
    ports:
      - 3000:3000
    expose:
      - 3000
    environment:
      - NODE_ENV=development
      - REACT_APP_SIM_HOST=http://localhost
      - REACT_APP_SIM_PORT=8000
      - REACT_APP_ARC_HOST=http://localhost
      - REACT_APP_ARC_PORT=80
      - REACT_APP_APM_URL=http://localhost/apm
      - REACT_APP_FLUENTD_URL=http://localhost:80/logger
    depends_on:
      - mongodb
      - simcct
      - fluentd
    networks:
      - dev_subnet

  # Nginx Microservice
  # This is an Nginx HTTP server that does reverse-proxy in the development
  # environment to easily access the `localhost` via URL matching to reverse
  # send traffic to the appropriate Docker container.
  nginx:
    build:
      context: ./services/nginx
      dockerfile: Dockerfile
      labels:
        arclytics.io: "development"
    labels:
      arclytics.io: "development"
    image: neuraldev/arc_sim_nginx:1.0
    logging:
      driver: fluentd
      options:
        fluentd-address: localhost:24224
        tag: nginx.docker.access
    restart: always
    ports:
      - 80:80
    depends_on:
      - fluentd
      - client
      - simcct
      - apm-server
    networks:
      - dev_subnet

  # SimCCT Microservice
  # This is the main service that provides user and admin management, authentication,
  # authorization, server-side session and cookie management, and the RESTful API
  # for the `arc_simulation` package. The `arc_simulation` package is a submodule
  # that provides the algorithm for phase transformation calculations. This service
  # is written with the Flask microframework and contains additional Flask plugins
  # provided by the open-source community.
  simcct:
    build:
      context: ./services/simcct
      dockerfile: Dockerfile
      labels:
        arclytics.io: "development"
    image: neuraldev/arc_sim_service:1.2
    volumes:
      - ./services/simcct:/usr/src/app  # This is a bind mount
    logging:
      driver: fluentd
      options:
        fluentd-address: localhost:24224
        tag: simcct.docker.access
    ports:
      - 8000:8000
    expose:
      - 8000
    environment:
      - FLASK_APP=sim_api/__init__.py
      - FLASK_ENV=development
      - APP_SETTINGS=configs.flask_conf.DevelopmentConfig
      - SECRET_KEY=${SECRET_KEY}
      - SECURITY_PASSWORD_SALT=${SECURITY_PASSWORD_SALT}
      - MONGO_HOST=mongodb
      - MONGO_PORT=27017
      - MONGO_APP_DB=arc_dev
      - MONGO_APP_USER=${MONGO_APP_USER}
      - MONGO_APP_USER_PASSWORD=${MONGO_APP_USER_PASSWORD}
      - REDIS_HOST=redis
      - REDIS_PORT=6379
      - REDIS_PASSWORD=${REDIS_PASSWORD}
      - CLIENT_SCHEME=http
      - CLIENT_HOST=localhost
      - CLIENT_PORT=80
      - DASK_SCHEDULER_ADDRESS=tcp://dask-scheduler:8786
      - FLUENTD_HOST=fluentd
      - FLUENTD_PORT=24224
      - FLUENTD_PREFIX_TAG=simcct
      - ELASTIC_APM_SERVER_URL=http://apm-server:8200
    depends_on:
      - mongodb
      - redis
      - fluentd
      - elasticsearch
      - apm-server
    networks:
      - dev_subnet

  # Arclytics Microservice
  # This is the container that provides analytics for the application. It ensures
  # authentication and authorization by using the same secret key as the `simcct`
  # microservice and decoding the JSON Web Token that is set in the cookie.
  # This service is written with the Flask microframework and contains additional
  # Flask plugins provided by the open-source community.
  arclytics:
    build:
      context: ./services/arclytics
      dockerfile: Dockerfile
      labels:
        arclytics.io: "development"
    image: neuraldev/arclytics_service:2.0
    logging:
      driver: fluentd
      options:
        fluentd-address: localhost:24224
        tag: arclytics.docker.access
    volumes:
      - ./services/arclytics:/usr/src/app
    ports:
      - 8001:8001
    expose:
      - 8001
    environment:
      - FLASK_APP=arc_api/__init__.py
      - FLASK_ENV=development
      - APP_SETTINGS=configs.flask_conf.DevelopmentConfig
      - SECRET_KEY=${SECRET_KEY}
      - SIMCCT_HOST=http://simcct:8000/v1/sim
      - REDIS_HOST=redis
      - REDIS_PORT=6379
      - REDIS_PASSWORD=${REDIS_PASSWORD}
      - MONGO_HOST=mongodb
      - MONGO_PORT=27017
      - MONGO_APP_DB=arc_dev
      - MONGO_APP_USER=${MONGO_APP_USER}
      - MONGO_APP_USER_PASSWORD=${MONGO_APP_USER_PASSWORD}
      - FLUENTD_HOST=fluentd
      - FLUENTD_PORT=24224
      - FLUENTD_PREFIX_TAG=arclytics
      - ELASTIC_APM_SERVER_URL=http://apm-server:8200
      - MAPBOX_TOKEN=${MAPBOX_TOKEN}
    depends_on:
      - fluentd
      - elasticsearch
    networks:
      - dev_subnet

  # Celery-Worker Microservice
  # This container contains a Flask instance of a Celery worker node that performs
  # asynchronous tasks based on the message queue from Redis and stores the result
  # in the Redis backend.
  celery-worker:
    build:
      context: ./services/celery-worker
      dockerfile: Dockerfile
      labels:
        arclytics.io: "development"
    image: neuraldev/arc_sim_celery:1.2
    logging:
      driver: fluentd
      options:
        fluentd-address: localhost:24224
        tag: celery.docker.access
    volumes:
      - ./services/celery-worker:/usr/src/app
    ports:
      - 5555:5555  # Flower Web GUI management of Celery
    expose:
      - 5555
    environment:
      - C_FORCE_ROOT=true
      - FLASK_ENV=development
      - APP_SETTINGS=flask_conf.DevelopmentConfig
      - MAIL_SERVER=${MAIL_SERVER}
      - MAIL_PORT=${MAIL_PORT}
      - MAIL_USE_TLS=${MAIL_USE_TLS}
      - MAIL_USERNAME=${MAIL_USERNAME}
      - MAIL_PASSWORD=${MAIL_PASSWORD}
      - MONGO_HOST=mongodb
      - MONGO_PORT=27017
      - MONGO_APP_DB=arc_dev
      - REDIS_HOST=redis
      - REDIS_PORT=6379
      - ELASTIC_APM_SERVER_URL=http://apm-server:8200
    depends_on:
      - redis
      - fluentd
      - elasticsearch
      - mongodb
    networks:
      - dev_subnet

  # Redis Microservice
  # This is a Redis in-memory data store and persistence layer than is used by other
  # services including `simcct`, `arclytics`, and `celery-worker`. It's main purpose
  # is for the server-side session storage for Flask and the message queue and results
  # backend for Celery.
  redis:
    build:
      context: ./services/redis
      dockerfile: Dockerfile
      labels:
        arclytics.io: "development"
    image: neuraldev/arc_sim_redis:1.1
    logging:
      driver: fluentd
      options:
        fluentd-address: localhost:24224
        tag: redis.docker.access
    restart: always
    ports:
      - 6379:6379
    expose:
      - 6379
    depends_on:
      - fluentd
    networks:
      - dev_subnet

  # MongoDB Microservice
  # MongoDB is an open-source, general purpose, document-based, distributed
  # NoSQL database. Our application uses MongoDB with a Replica Set distribution
  # as the data persistence layer for our users and application data. We also
  # use MongoDB’s querying language and pipelines to leverage data analytics
  # about our users, application, and the simulation data.
  mongodb:
    build:
      context: ./services/db
      dockerfile: Dockerfile
      labels:
        arclytics.io: "development"
    image: neuraldev/arc_sim_mongo:1.1
    logging:
      driver: fluentd
      options:
        fluentd-address: localhost:24224
        tag: mongodb.docker.access
    restart: always
    volumes:
      - ./services/db/test_data:/data/test
    ports:
      - 27017:27017
    expose:
      - 27017
    environment:
      - MONGO_DATA_DIR=/usr/data/db
      - MONGO_LOG_DIR=/dev/null
    depends_on:
      - fluentd
    networks:
      - dev_subnet

  # Elasticsearch Microservice
  # Elasticsearch is a distributed, open-source RESTful, JSON-based search and
  # analytics engine. Whilst Elasticsearch as a tool provides many features,
  # one of it’s common use cases is the foundation of the ELK stack (acronym
  # for “Elasticsearch,” “Logstash,” and “Kibana”) for logging and logging
  # analytics. Arclytics SimCCT, being a distributed microservices architecture,
  # solved the problem of distributed container logs by building a unified
  # logging stack for rapid development debugging and deployment maintenance
  # and metrics. Elasticsearch is the foundation of this stack.
  elasticsearch:
    build:
      context: ./services/elasticsearch
      dockerfile: Dockerfile
      labels:
        arclytics.io: "development"
    image: neuraldev/arc_elasticsearch:latest
    volumes:
      - elasticsearch-data:/usr/share/elasticsearch/data
    ports:
      - 9200:9200
      - 9300:9300
    expose:
      - 9200
      - 9300
    environment:
      - discovery.type=single-node
      - cluster.name=arc-sim-cluster
      - bootstrap.memory_lock=false
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    networks:
      - dev_subnet

  # APM Server Microservice
  # Elastic’s Application Performance Monitoring (APM) tool allows our software
  # team to better understand how the Arclytics SimCCT application is performing
  # in real-time. One of it’s core features is providing distributed and
  # reliable tracing of our API services and the ability to monitor and capture
  # bugs and exceptions. Due to the nature of our distributed microservices, APM
  # has been extremely powerful in allowing us to find bottlenecks and bugs
  # faster so that we can build an application that is of a higher-quality for
  # our client and users.
  apm-server:
    build:
      context: ./services/apm-server
      dockerfile: Dockerfile
      labels:
        arclytics.io: "development"
    image: neuraldev/arc_apm_server:latest
    logging:
      driver: fluentd
      options:
        fluentd-address: localhost:24224
        tag: apm.docker.access
    ports:
      - 8200:8200
    expose:
      - 8200
    volumes:
      - ./services/apm-server/apm-server.docker.yml:/usr/share/apm-server/apm-server.yml
    depends_on:
      - elasticsearch
    networks:
      - dev_subnet

  # Kibana Microservice
  # Kibana is a data visualization and management tool for Elasticsearch that
  # provides real-time metrics. It mainly allows our software team to visualise
  # and navigate the EFK stack whether we are in a development or production
  # environment.
  kibana:
    build:
      context: ./services/kibana
      dockerfile: Dockerfile
      labels:
        arclytics.io: "development"
    image: neuraldev/arc_kibana:latest
    ports:
      - 5601:5601
    expose:
      - 5601
    volumes:
      - ./services/kibana/kibana.yml:/user/share/kibana/config/kibana.yml
    depends_on:
      - elasticsearch
      - fluentd
    networks:
      - dev_subnet

  # Swagger Microservice
  # Swagger is an open-source Application Programming Interface development tool
  # that follows the OpenAPI standard. In our development environment, we deploy
  # a Swagger service that allows us to document and test our RESTful API design
  # and implementation.
  swagger:
    build:
      context: ./services/swagger
      dockerfile: Dockerfile
      labels:
        arclytics.io: "development"
    image: neuraldev/arc_sim_swagger:1.0
    ports:
      - 3001:8080  # expose ports - HOST:CONTAINER
    expose:
      - 3001
    environment:
      - URL=swagger.yaml
    networks:
      - dev_subnet

  # ============================== # DASK MICROSERVICES # ============================== #
  dask-scheduler:
    build:
      context: ./services/dask
      dockerfile: Dockerfile
      labels:
        arclytics.io: "development"
    image: neuraldev/arc_dask_scheduler:1.0
    hostname: dask-scheduler
    ports:
      - 8786:8786
      - 8787:8787
    expose:
      - 8786
      - 8787
    # Entrypoint command needs to be set here because Dockerfile shared with `dask-worker`
    command: ['dask-scheduler']
    networks:
      - dev_subnet

  dask-worker:
    build:
      context: ./services/simcct
      dockerfile: Dockerfile-worker
      labels:
        arclytics.io: "development"
    image: neuraldev/arc_dask_worker:1.0
    hostname: dask-worker
    command: ['dask-worker', 'tcp://dask-scheduler:8786', '--no-nanny']
    networks:
      - dev_subnet

  jupyter:
    build:
      context: ./services/jupyter
      dockerfile: Dockerfile
      labels:
        arclytics.io: "development"
    image: neuraldev/arc_dask_notebook:1.0
    domainname: io.arclytics.app
    hostname: jupyter
    volumes:
      - type: bind
        source: ./services/jupyter/notebooks
        target: /home/arclytics/notebooks
    ports:
      - 8888:8888
    expose:
      - 8888
    environment:
      - NB_USER=arclytics
      - NB_UID=1000
      - NB_GID=100
      - NB_UMASK=002
      - JUPYTER_ENABLE_LAB=yes
      - DASK_SCHEDULER_ADDRESS=tcp://dask-scheduler:8786
      - MONGO_URI=mongodb://mongodb:27017
      - REDIS_URI=redis://redis:6379
    networks:
      - dev_subnet

  # ============================== # TEST MICROSERVICES # ============================== #
  # Both these services were created for testing purposes and a proof-of-concept for
  # the use of `fluentd` as the logging driver for Docker. They are not to be used
  # in any production environment as they are not configured at all.
  # ==================================================================================== #
  fluent-python:
    build:
      context: ./services/fluent-python
      dockerfile: Dockerfile
      labels:
        arclytics.io: "concept"
    image: neuraldev/arc_fluent_python:1.0
    ports:
      - 5005:5000
    expose:
      - 5005
    logging:
      driver: fluentd
      options:
        fluentd-address: localhost:24224
        tag: fluent-python.access
    environment:
      - FLUENT_HOST=fluentd
    depends_on:
      - fluentd
    networks:
      - dev_subnet

  fluent-react:
    build:
      context: ./services/fluent-react
      dockerfile: Dockerfile
      labels:
        arclytics.io: "concept"
    image: neuraldev/arc_fluent_react:1.0
    ports:
      - 80:80
    expose:
      - 80
    logging:
      driver: fluentd
      options:
        fluentd-address: localhost:24224
        tag: fluent-react.access
    depends_on:
      - fluentd
    networks:
      - dev_subnet
  # =================================================================================== #

networks:
  dev_subnet:
    driver: bridge
    labels:
      arclytics.io: "development"

# https://docs.docker.com/compose/compose-file/#volume-configuration-reference
volumes:
  elasticsearch-data:
    labels:
      arclytics.io: "development"
      service: "elasticsearch"
    driver: local
#  DECISION:
#  This is not working as a dir needs to be created before binding the mount.
#  Since Docker creates a Volume in /var/lib/docker/volumes/vol_name/_data
#  which is an abstraction layer that gets bound to the Docker container.
#  We prefer to use bind mounts in development so that we can easily destroy
#  data quickly and bring up new ones using scripts for MongoDB and Redis.
#  arclytics_dir:
#    labels:
#      arclytics.io: "development"
#    driver: local
#    driver_opts:
#      device: "./services/users"
#      type: none
#      o: bind