Feature/image size optimisation (#13)

Miggets7 · web-flow · commit 73af8f7c6ae5 · 2026-01-16T13:54:46.000Z
diff --git a/.github/workflows/postgresql.yml b/.github/workflows/postgresql.yml
@@ -92,16 +92,34 @@ jobs:
           username: ${{ secrets._TEMP_DOCKERHUB_USER }}
           password: ${{ secrets._TEMP_DOCKERHUB_PASSWORD }}
         
-      - name: build and push amd64 image
+      - name: Build amd64 image locally
         run: |
           docker buildx build \
             --build-arg GIT_COMMIT=${{ github.sha }} \
-            --push \
+            --load \
             --platform linux/amd64 \
             --no-cache-filter trimmed \
             --no-cache-filter trimmed-all \
             $TAG .
 
+      - name: Install slim toolkit
+        run: |
+          curl -sL https://raw.githubusercontent.com/slimtoolkit/slim/master/scripts/install-slim.sh | sudo -E bash -
+
+      - name: Slim the image
+        run: |
+          # Extract image name from TAG (remove -t prefix)
+          IMAGE_NAME=$(echo "$TAG" | sed 's/-t //')
+          chmod +x ./slim-image.sh
+          ./slim-image.sh "$IMAGE_NAME" "${IMAGE_NAME}-slim" amd64
+          # Replace original with slim version
+          docker tag ${IMAGE_NAME}-slim $IMAGE_NAME
+
+      - name: Push amd64 image
+        run: |
+          IMAGE_NAME=$(echo "$TAG" | sed 's/-t //')
+          docker push $IMAGE_NAME
+
   image_postgresql_arm64:
     needs: image_postgresql_amd64
     runs-on: ubuntu-latest
@@ -171,16 +189,34 @@ jobs:
           username: ${{ secrets._TEMP_DOCKERHUB_USER }}
           password: ${{ secrets._TEMP_DOCKERHUB_PASSWORD }}
         
-      - name: build and push arm64 image
+      - name: Build arm64 image locally
         run: |
           docker buildx build \
             --build-arg GIT_COMMIT=${{ github.sha }} \
-            --push \
-            --platform linux/aarch64 \
+            --load \
+            --platform linux/arm64 \
             --no-cache-filter trimmed \
             --no-cache-filter trimmed-all \
             $TAG .
 
+      - name: Install slim toolkit
+        run: |
+          curl -sL https://raw.githubusercontent.com/slimtoolkit/slim/master/scripts/install-slim.sh | sudo -E bash -
+
+      - name: Slim the image
+        run: |
+          # Extract image name from TAG (remove -t prefix)
+          IMAGE_NAME=$(echo "$TAG" | sed 's/-t //')
+          chmod +x ./slim-image.sh
+          ./slim-image.sh "$IMAGE_NAME" "${IMAGE_NAME}-slim" arm64
+          # Replace original with slim version
+          docker tag ${IMAGE_NAME}-slim $IMAGE_NAME
+
+      - name: Push arm64 image
+        run: |
+          IMAGE_NAME=$(echo "$TAG" | sed 's/-t //')
+          docker push $IMAGE_NAME
+
   create_manifest:
     needs: [image_postgresql_amd64, image_postgresql_arm64]
     runs-on: ubuntu-latest
diff --git a/Dockerfile b/Dockerfile
@@ -1,51 +1,78 @@
-
 ARG PG_MAJOR=17
+ARG PREV_PG_MAJOR=15
 ARG TIMESCALE_VERSION=2.22
 
-FROM timescale/timescaledb-ha:pg17-ts${TIMESCALE_VERSION} AS trimmed
-LABEL maintainer="support@openremote.io"
+# Stage 1: Get PostgreSQL ${PREV_PG_MAJOR} binaries for upgrade support
+FROM timescale/timescaledb-ha:pg${PG_MAJOR}-ts${TIMESCALE_VERSION}-all AS pg-all
 
 USER root
 
-# install fd to find files to speed up chown and chgrp
-RUN apt-get update && apt-get install -y fd-find && rm -rf /var/lib/apt/lists/*
-
-# Give postgres user the same UID and GID as the old alpine postgres image to simplify migration of existing DB
-RUN usermod -u 70 postgres \
- && groupmod -g 70 postgres \
- && (fd / -group 1000 -exec chgrp -h postgres {} \; || true) \
- && (fd / -user 1000 -exec chown -h postgres {} \; || true)
+ARG PREV_PG_MAJOR
 
-# Set PGDATA to the same location as our old alpine image
-RUN mkdir -p /var/lib/postgresql && mv /home/postgres/pgdata/* /var/lib/postgresql/ && chown -R postgres:postgres /var/lib/postgresql
+# Strip debug symbols and remove unnecessary files from PG ${PREV_PG_MAJOR} in this stage
+# For pg_upgrade we need bin/, lib/, and extension/ (for TimescaleDB upgrade scripts)
+RUN find /usr/lib/postgresql/${PREV_PG_MAJOR} -type f -name '*.so*' -exec strip --strip-unneeded {} \; 2>/dev/null || true \
+    && find /usr/lib/postgresql/${PREV_PG_MAJOR} -type f -executable -exec strip --strip-unneeded {} \; 2>/dev/null || true \
+    && rm -rf /usr/share/postgresql/${PREV_PG_MAJOR}/man \
+              /usr/share/postgresql/${PREV_PG_MAJOR}/doc \
+              /usr/share/postgresql/${PREV_PG_MAJOR}/contrib
 
-# Add custom entry point (see file header for details)
-COPY or-entrypoint.sh /
-RUN chmod +x /or-entrypoint.sh
+# Stage 2: Prepare the main image with UID/GID changes and cleanup
+FROM timescale/timescaledb-ha:pg${PG_MAJOR}-ts${TIMESCALE_VERSION} AS final
+LABEL maintainer="support@openremote.io"
 
-# Add custom initdb script(s)
-COPY docker-entrypoint-initdb.d/ /docker-entrypoint-initdb.d/
-RUN chmod +x /docker-entrypoint-initdb.d/*
+USER root
 
+ARG PREV_PG_MAJOR
 
-# Below is mostly copied from https://github.com/timescale/timescaledb-docker-ha/blob/master/Dockerfile (with OR specific entrypoint,
-# workdir and OR env defaults)
+# Copy PG ${PREV_PG_MAJOR} bin and lib directories for pg_upgrade
+COPY --from=pg-all /usr/lib/postgresql/${PREV_PG_MAJOR}/bin /usr/lib/postgresql/${PREV_PG_MAJOR}/bin
+COPY --from=pg-all /usr/lib/postgresql/${PREV_PG_MAJOR}/lib /usr/lib/postgresql/${PREV_PG_MAJOR}/lib
+# Copy share files including extensions (needed for TimescaleDB upgrade on old PG before pg_upgrade)
+COPY --from=pg-all /usr/share/postgresql/${PREV_PG_MAJOR} /usr/share/postgresql/${PREV_PG_MAJOR}
 
-# Get the -all variant which contains multiple PostgreSQL versions
-# According to TimescaleDB docs: "timescale/timescaledb-ha images have the files necessary to run previous versions"
-FROM timescale/timescaledb-ha:pg17-ts${TIMESCALE_VERSION}-all AS trimmed-all
+# Copy entrypoint scripts
+COPY or-entrypoint.sh /
+COPY docker-entrypoint-initdb.d/ /docker-entrypoint-initdb.d/
 
-## Create a smaller Docker image from the builder image
-FROM scratch
-COPY --from=trimmed / /
+# Install fd-find, fix UID/GID, setup directories, strip binaries, and cleanup - all in one layer
+RUN apt-get update && apt-get install -y --no-install-recommends fd-find \
+    # Give postgres user the same UID and GID as the old alpine postgres image
+    && usermod -u 70 postgres \
+    && groupmod -g 70 postgres \
+    && (fdfind . / -group 1000 -exec chgrp -h postgres {} \; 2>/dev/null || true) \
+    && (fdfind . / -user 1000 -exec chown -h postgres {} \; 2>/dev/null || true) \
+    # Set PGDATA to the same location as our old alpine image
+    && mkdir -p /var/lib/postgresql \
+    && mv /home/postgres/pgdata/* /var/lib/postgresql/ \
+    && chown -R postgres:postgres /var/lib/postgresql \
+    # Make scripts executable
+    && chmod +x /or-entrypoint.sh /docker-entrypoint-initdb.d/* \
+    # Strip debug symbols from PostgreSQL binaries to reduce size
+    && find /usr/lib/postgresql -type f -name '*.so*' -exec strip --strip-unneeded {} \; 2>/dev/null || true \
+    && find /usr/lib/postgresql -type f -executable -exec strip --strip-unneeded {} \; 2>/dev/null || true \
+    # Remove fd-find and clean up
+    && apt-get purge -y fd-find \
+    && apt-get autoremove -y --purge \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* \
+              /var/cache/apt/* \
+              /var/log/* \
+              /usr/share/doc/* \
+              /usr/share/man/* \
+              /usr/share/info/* \
+              /usr/share/lintian/* \
+              /usr/share/locale/* \
+              /tmp/* \
+              /var/tmp/* \
+              /root/.cache \
+              /home/postgres/.cache \
+              /usr/local/lib/pgai \
+              /usr/share/postgresql/*/man \
+              /usr/share/postgresql/*/doc
 
 ARG PG_MAJOR
-
-## Copy only PostgreSQL 14 and 15 for upgrade support
-COPY --from=trimmed-all /usr/lib/postgresql/14 /usr/lib/postgresql/14
-COPY --from=trimmed-all /usr/lib/postgresql/15 /usr/lib/postgresql/15
-COPY --from=trimmed-all /usr/share/postgresql/14 /usr/share/postgresql/14
-COPY --from=trimmed-all /usr/share/postgresql/15 /usr/share/postgresql/15
+ARG PREV_PG_MAJOR
 
 # Increment this to indicate that a re-index should be carried out on first startup with existing data; REINDEX can still be overidden
 # with OR_DISABLE_REINDEX=true
@@ -80,6 +107,7 @@ ENV PGROOT=/var/lib/postgresql \
     POSTGRES_USER=${POSTGRES_USER:-postgres} \
     POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} \
     PG_MAJOR=$PG_MAJOR \
+    PREV_PG_MAJOR=$PREV_PG_MAJOR \
     OR_REINDEX_COUNTER=${OR_REINDEX_COUNTER} \
     OR_DISABLE_REINDEX=${OR_DISABLE_REINDEX:-false} \
     POSTGRES_MAX_CONNECTIONS=${POSTGRES_MAX_CONNECTIONS:-50} \
diff --git a/README.md b/README.md
@@ -1,33 +1,99 @@
 # Postgresql docker image
-[![build multirach postgresql Docker image and push to it dockerhub](https://github.com/openremote/postgresql/actions/workflows/postgresql.yml/badge.svg)](https://github.com/openremote/postgresql/actions/workflows/postgresql.yml)
+[![build multiarch postgresql Docker image and push to dockerhub](https://github.com/openremote/postgresql/actions/workflows/postgresql.yml/badge.svg)](https://github.com/openremote/postgresql/actions/workflows/postgresql.yml)
 
- POSTGIS and TimescaleDB (inc. toolkit for hyperfunctions) image built for aarch64 support using `timescaledev/timescaledb-ha` base image with:
+POSTGIS and TimescaleDB (inc. toolkit for hyperfunctions) image built for amd64 and arm64 using `timescale/timescaledb-ha` base image with:
 
 - OR specific ENV variables and a healthcheck added
 - Easy configuration of `max_connections` using `POSTGRES_MAX_CONNECTIONS` environment variable (set to `-1` to disable this setting)
 - PGDATA path set to match old Alpine image (for ease of DB migration)
 - POSTGRES user UID and GID changed to match old Alpine image (for ease of DB migration)
 - Auto upgrade of database with PG major version changes from previous PG major version; can be disabled using
-  OR_DISABLE_AUTO_UPGRADE=true.
+  `OR_DISABLE_AUTO_UPGRADE=true`
 - Auto upgrade of timescaleDB extension when a new version is available in the container; can be disabled using
-  OR_DISABLE_AUTO_UPGRADE=true.
+  `OR_DISABLE_AUTO_UPGRADE=true`
 - OR_DISABLE_REINDEX env variable with associated scripts to determine if a REINDEX of the entire DB should be carried
-  out at first startup with existing DB (checks whether or not $PGDATA/OR_REINDEX_COUNTER.$OR_REINDEX_COUNTER exists).
+  out at first startup with existing DB (checks whether or not `$PGDATA/OR_REINDEX_COUNTER.$OR_REINDEX_COUNTER` exists).
   This is used when a collation change has occurred (glibc version change, muslc <-> glibc) which can break the indexes;
   migration can either be manually handled or auto handled depending on OR_DISABLE_REINDEX env variable value.
-  NOTE THAT A REINDEX CAN TAKE A LONG TIME DEPENDING ON THE SIZE OF THE DB! And startup will be delayed until completed
+  NOTE THAT A REINDEX CAN TAKE A LONG TIME DEPENDING ON THE SIZE OF THE DB! And startup will be delayed until completed.
   This functionality is intended to simplify migration for basic users; advanced users with large DBs should take care of this
   themselves.
+- **Slimmed images** using [slim toolkit](https://github.com/slimtoolkit/slim) to reduce image size by ~60%
 
-`timescale/timescaledb-ha` image is ubuntu based and only currently supports amd64; they are working on ARM64 support in timescaledev/timescaledb-ha see:
+## Local Development
 
-https://github.com/timescale/timescaledb-docker-ha/pull/355
+### Prerequisites
 
-See this issue for POSTGIS base image aarch64 support discussion:
+- **Docker** must be installed and running
+- **slim toolkit** must be installed for image optimization
 
-https://github.com/postgis/docker-postgis/issues/216
+Install slim toolkit via the install script:
+```bash
+curl -sL https://raw.githubusercontent.com/slimtoolkit/slim/master/scripts/install-slim.sh | sudo -E bash -
+```
 
-TODO: Switch over to timescale/timescaledb-ha once arm64 supported
+Or via Homebrew (macOS):
+```bash
+brew install docker-slim
+```
+
+For more installation options, see the [slim toolkit documentation](https://github.com/slimtoolkit/slim#installation).
+
+### Building the Image
+
+1. Build the Docker image (replace `17` with desired PostgreSQL major version):
+```bash
+docker build --build-arg PG_MAJOR=17 -t openremote/postgresql:pg17 .
+```
+
+2. Slim the image using the provided script:
+```bash
+# Usage: ./slim-image.sh <source-image> <target-image> <architecture>
+# Architecture: amd64 or arm64 (auto-detected if omitted)
+
+./slim-image.sh openremote/postgresql:pg17 openremote/postgresql:pg17-slim
+```
+
+3. Optionally replace the original with the slimmed version:
+```bash
+docker tag openremote/postgresql:pg17-slim openremote/postgresql:pg17
+```
 
 ## Upgrading
+
 ***NOTE: If you change the version of container you use then make sure you have backed up your DB first as this container will try to auto upgrade your DB and/or TimescaleDB extension; this auto upgrade functionality can be disabled using `OR_DISABLE_AUTO_UPGRADE=true`***
+
+### Automatic Upgrade
+
+This image supports automatic upgrades from the previous PostgreSQL major version. When the container starts with an existing database from a supported older version, it will:
+
+1. Upgrade TimescaleDB extensions on the old PostgreSQL version
+2. Run `pg_upgrade` to migrate the database to the new PostgreSQL version
+3. Upgrade TimescaleDB extensions on the new PostgreSQL version
+
+### Manual Upgrade
+
+If automatic upgrade is not supported for your database version (e.g., skipping multiple major versions), you will need to perform a manual upgrade. Follow these steps:
+
+1. **Backup your database** using `pg_dump` or `pg_dumpall`
+2. **Upgrade TimescaleDB first** (if installed) - this must be done before PostgreSQL upgrade
+3. **Use pg_upgrade** to migrate between PostgreSQL versions, or restore from backup to a fresh database
+
+#### Useful Resources
+
+- [PostgreSQL pg_upgrade documentation](https://www.postgresql.org/docs/current/pgupgrade.html)
+- [TimescaleDB upgrade guide](https://docs.timescale.com/self-hosted/latest/upgrades/)
+- [TimescaleDB major upgrade guide](https://docs.timescale.com/self-hosted/latest/upgrades/major-upgrade/)
+
+#### Example: Manual pg_dump/restore
+
+```bash
+# On the old container, dump the database
+docker exec -it <old_container> pg_dumpall -U postgres > backup.sql
+
+# Start the new container with a fresh data directory
+docker run -d --name new_postgres -v /path/to/new/data:/var/lib/postgresql/data openremote/postgresql:latest
+
+# Restore the backup
+docker exec -i new_postgres psql -U postgres < backup.sql
+```
diff --git a/or-entrypoint.sh b/or-entrypoint.sh
@@ -77,6 +77,27 @@ if [ -n "$DATABASE_ALREADY_EXISTS" ]; then
       echo "---------------------------------------------------------------------------------"
     fi
 
+    # Check if the old DB version is supported for upgrade
+    if [ "$DB_VERSION" != "$PG_MAJOR" ] && [ "$OR_DISABLE_AUTO_UPGRADE" != "true" ]; then
+      # Only PREV_PG_MAJOR and PG_MAJOR are supported
+      if [ "$DB_VERSION" != "$PREV_PG_MAJOR" ] && [ "$DB_VERSION" != "$PG_MAJOR" ]; then
+        echo "********************************************************************************"
+        echo "ERROR: Database version ${DB_VERSION} is not supported for automatic upgrade!"
+        echo "This image only supports upgrading from PostgreSQL ${PREV_PG_MAJOR} to ${PG_MAJOR}."
+        echo ""
+        echo "Options:"
+        echo "  1. Use an intermediate image version that supports upgrading from ${DB_VERSION}"
+        echo "  2. Manually upgrade the database (see documentation below)"
+        echo ""
+        echo "Documentation:"
+        echo "  - OpenRemote PostgreSQL upgrade guide: https://github.com/openremote/postgresql#upgrading"
+        echo "  - PostgreSQL pg_upgrade: https://www.postgresql.org/docs/current/pgupgrade.html"
+        echo "  - TimescaleDB upgrade guide: https://docs.timescale.com/self-hosted/latest/upgrades/"
+        echo "********************************************************************************"
+        exit 12
+      fi
+    fi
+
     # STEP 1: Upgrade TimescaleDB on OLD PostgreSQL version (if needed)
     # This must happen BEFORE pg_upgrade so both old and new PG have the same TS version
     if [ "$DB_VERSION" != "$PG_MAJOR" ] && [ "$OR_DISABLE_AUTO_UPGRADE" != "true" ]; then
diff --git a/slim-image.sh b/slim-image.sh
