Skip to content

Adding VersionStream for superset-6.0 #78596

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
octo-sts wants to merge 1 commit into
base: main
Choose a base branch
from
+309 −0
Open

Adding VersionStream for superset-6.0 #78596

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
309 changes: 309 additions & 0 deletions superset-6.0.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,309 @@
package:
name: superset-6.0
version: "6.0.0"
epoch: 0 # GHSA-63vm-454h-vhhq
description: Data Visualization and Data Exploration Platform
copyright:
- license: Apache-2.0
resources:
cpu: 65
memory: 128Gi
options:
# There is a dependency on libarrow-substrait.so although it
# is provided in the virtual environment. Enabling no-depends
# works around this
no-depends: true
no-provides: true
dependencies:
runtime:
- ecpg
- libgomp
- libtbb-dev
- openssl
- py${{vars.python-version}}-pip
provides:
- superset

var-transforms:
- from: ${{package.version}}
match: ^(\d+).*
replace: $1
to: major-version

environment:
contents:
packages:
- build-base
- busybox
- ca-certificates-bundle
- mariadb-connector-c-dev
- mariadb-dev
- nodejs-20 # Fixes an unsupported engine error
- npm<11.0.0 # Fixes an unsupported engine error
- nvm
- openldap-dev
- patch
- py${{vars.python-version}}-pip
- py${{vars.python-version}}-sqlparse
- python-${{vars.python-version}}-dev
- zlib-dev
- zstd

vars:
python-version: 3.11

pipeline:
- uses: git-checkout
with:
repository: https://github.com/apache/superset.git
tag: ${{package.version}}
expected-commit: 6a1c30e5e7c3e28d0549c9c2ac0ff61607f26a2f

- name: Build frontend
working-directory: superset-frontend
runs: |
# npm ci does not work as 4.1.2 provides package-lock.json from 4.1.1
npm install
npm run build
mkdir -p ${{targets.destdir}}/app/superset/static/assets
cp -r ../superset/static/assets/* ${{targets.destdir}}/app/superset/static/assets

- name: Build backend
runs: |
python -m venv venv --system-site-packages
source venv/bin/activate

# To install mysqlclient wheel
export MYSQLCLIENT_CFLAGS=`mysql_config --cflags`
export MYSQLCLIENT_LDFLAGS=`mysql_config --libs`
pip install -r requirements/base.txt

# To fix vulnerabilities
pip install --upgrade dnspython==2.6.1 gunicorn==23.0.0 idna==3.7 setuptools==78.1.1 sqlparse==0.5.0 Jinja2==3.1.6 Werkzeug==3.1.5 requests==2.32.4 urllib3==2.6.3 certifi==2024.07.04 zipp==3.19.2 pillow==11.3.0 brotli==1.2.0 marshmallow==3.26.2 pynacl==1.6.2 pyasn1==0.6.2

# Dependencies required during runtime
pip install pillow pyarrow

# For running translations
pip install flask flask-appbuilder==4.8.1

sed -i 's/"cryptography>=42.0.4.*"/"cryptography>=44.0.1"/' pyproject.toml

# Build Apache Superset
pip install .

# Install Postgres pattern
pip install .[postgres]

# translations for the web application, Is a non-blocking call, so errors may be ignored
flask fab babel-compile --target superset/translations

# Remove malware-scan-triggering tests folder from built package
rm -rf venv/lib/python${{vars.python-version}}/site-packages/tests/

# Remove pip
pip uninstall --yes pip

# Install virtual environment
mkdir -p ${{targets.destdir}}/usr/share/superset
mv venv ${{targets.destdir}}/usr/share/superset

# Remove pycache
rm -rf ${{targets.destdir}}/usr/share/superset/venv/bin/__pycache*

# Fix virtual environment's installed path
sed -i "s|/home/build|/usr/share/superset|g" ${{targets.destdir}}/usr/share/superset/venv/bin/*
sed -i "s|/home/build|/usr/share/superset|g" ${{targets.destdir}}/usr/share/superset/venv/pyvenv.cfg

# Install frontend
mkdir -p ${{targets.destdir}}/app/superset-frontend
cp -r ./superset ${{targets.destdir}}/app/
cp setup.py MANIFEST.in README.md ${{targets.destdir}}/app/
cp superset-frontend/package.json ${{targets.destdir}}/app/superset-frontend/

subpackages:
- name: ${{package.name}}-entrypoint
description: "Docker configuration for Superset"
dependencies:
runtime:
- bash
- busybox
- coreutils
pipeline:
- runs: |
mkdir -p ${{targets.contextdir}}/usr/bin/
cp ./docker/entrypoints/run-server.sh ${{targets.contextdir}}/usr/bin/
chmod 755 ${{targets.contextdir}}/usr/bin/run-server.sh

# Docker bootstrap scripts
mkdir -p ${{targets.contextdir}}/app/docker
cp ./docker/docker-bootstrap.sh ${{targets.contextdir}}/app/docker/
cp ./docker/docker-frontend.sh ${{targets.contextdir}}/app/docker/
cp ./docker/docker-init.sh ${{targets.contextdir}}/app/docker/
cp ./docker/frontend-mem-nag.sh ${{targets.contextdir}}/app/docker/
chmod 755 ${{targets.contextdir}}/app/docker/*.sh

# Entrypoint scripts
mkdir -p ${{targets.contextdir}}/app/docker/entrypoints
cp ./docker/entrypoints/run-server.sh ${{targets.contextdir}}/app/docker/entrypoints/
cp ./docker/entrypoints/docker-ci.sh ${{targets.contextdir}}/app/docker/entrypoints/
chmod 755 ${{targets.contextdir}}/app/docker/entrypoints/*.sh
test:
pipeline:
- runs: |
if ! [ -x /app/docker/entrypoints/run-server.sh ]; then
echo "Entrypoint isn't executable" && exit 1
fi

- name: ${{package.name}}-iamguarded-compat
description: "compat package for iamguarded"
dependencies:
runtime:
# Required by startup scripts
- bash
- coreutils
- posix-libc-utils
pipeline:
- uses: iamguarded/build-compat
with:
package: superset
version: ${{vars.major-version}}
- runs: |
ln -s /app/superset /opt/iamguarded/superset

# copy config file
mkdir -p ${{targets.contextdir}}/app/superset
install -m755 /opt/iamguarded/scripts/superset/files/superset_config.py ${{targets.contextdir}}/app/superset/

# Fix user config - we need this to allow superset to be run
# as root inside bubblewrap, otherwise we can't test on aarch64
sed -i 's~export SUPERSET_DAEMON_USER="superset"~export SUPERSET_DAEMON_USER=${SUPERSET_DAEMON_USER:-"superset"}~g' /opt/iamguarded/scripts/superset-env.sh
sed -i 's~export SUPERSET_DAEMON_GROUP="superset"~export SUPERSET_DAEMON_GROUP=${SUPERSET_DAEMON_GROUP:-"superset"}~g' /opt/iamguarded/scripts/superset-env.sh

cat << EOF > run.patch
--- /opt/iamguarded/scripts/superset/run.sh
+++ run.sh
@@ -22,6 +22,17 @@
. /opt/iamguarded/scripts/libos.sh
. /opt/iamguarded/scripts/libsuperset.sh

+should_i_switch_user(){
+ if am_i_root; then
+ if [[ "\$SUPERSET_DAEMON_USER" != "root" ]]; then
+ # Switch User
+ return 0
+ fi
+ fi
+ # Return false
+ return 1
+}
+
if [[ "\$SUPERSET_ROLE" = "webserver" ]]; then
command="gunicorn"
args=(
@@ -55,7 +66,7 @@
fi

info "** Starting Superset \${SUPERSET_ROLE} **"
-if am_i_root; then
+if should_i_switch_user; then
exec_as_user "\$SUPERSET_DAEMON_USER" "\${command}" "\${args[@]}"
else
exec "\${command}" "\${args[@]}"
EOF

patch /opt/iamguarded/scripts/superset/run.sh < run.patch
- uses: iamguarded/finalize-compat
with:
package: superset
version: ${{vars.major-version}}
test:
environment:
contents:
packages:
- superset
- bash
- coreutils
- posix-libc-utils
- cmd:useradd
pipeline:
- uses: iamguarded/test-compat
with:
package: superset
version: ${{vars.major-version}}
- uses: test/daemon-check-output
with:
setup: |
set -eu
groupadd superset ; useradd -g superset superset
mkdir -p /opt/iamguarded/superset/tmp ; chmod 0777 /opt/iamguarded/superset/tmp
mkdir -p /opt/iamguarded/superset/logs ; chmod 0777 /opt/iamguarded/superset/logs
mkdir -p /opt/iamguarded/superset/superset_home ; chmod 0777 /opt/iamguarded/superset/superset_home
start: |
env \
SUPERSET_DAEMON_USER=root \
SUPERSET_DAEMON_GROUP=root \
SUPERSET_DATABASE_HOST=localhost \
SUPERSET_HOME=/app/superset_home \
PATH=/usr/share/superset/venv/bin:$PATH \
SUPERSET_SECRET_KEY=1234 \
SUPERSET_ROLE=webserver \
FLASK_APP='superset.app:create_app()' \
PYTHONPATH=/app/pythonpath \
bash \
/opt/iamguarded/scripts/superset/entrypoint.sh \
/opt/iamguarded/scripts/superset/run.sh
timeout: 60
expected_output: |
Welcome to the Iamguarded image for Superset
Starting Superset setup
Initializing Superset
Superset setup finished!
Starting Superset webserver
Starting gunicorn 23.0.0
Listening at: http://0.0.0.0:8080
Using worker: gthread
Booting worker with pid
# We expect the worker to ERROR as there isn't a database available
error_strings: |
FAIL
FATAL
Traceback.*most.recent.call
Exception in thread
command not found

update:
enabled: true
ignore-regex-patterns:
- '^superset-helm-chart'
- 'rc\d+$'
github:
identifier: apache/superset
tag-filter-prefix: "6.0"

test:
environment:
contents:
packages:
- ${{package.name}}-entrypoint
pipeline:
- runs: |
ecpg --help
- uses: test/tw/ldd-check
- uses: test/daemon-check-output
with:
start: |
env \
PATH=/usr/share/superset/venv/bin:$PATH \
SUPERSET_ENV=production \
FLASK_APP="superset.app:create_app()" \
SUPERSET_PORT=8088 \
SUPERSET_SECRET_KEY="$(openssl rand -base64 42)" \
/app/docker/entrypoints/docker-ci.sh > /dev/null 2>&1
timeout: 60
expected_output: |
Applying DB migrations
Setting up admin user
Setting up roles and perms
Starting gunicorn
Listening at: http://0.0.0.0:8088
Loading