diff --git a/cookiecutter-pypackage/.github/workflows/build.yml b/cookiecutter-pypackage/.github/workflows/build.yml new file mode 100644 index 0000000..acd402d --- /dev/null +++ b/cookiecutter-pypackage/.github/workflows/build.yml @@ -0,0 +1,99 @@ +name: Build + +on: + workflow_call: + inputs: + vpu_docker_registry: + required: true + type: string + vpu_docker_image_name: + required: true + type: string + vpu_docker_repositories: + required: true + type: string + build_name: + required: true + type: string + build_number: + required: true + type: string + +jobs: + build-python: + runs-on: ubuntu-latest + env: + COLUMNS: 160 + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - uses: astral-sh/setup-uv@v7 + + - name: Sync project + run: uv sync --frozen + + - name: Build docs + run: uv run tox run -e docs + + - name: Build python app + run: uv run tox run -e build + + - name: Upload python artifacts + uses: actions/upload-artifact@v7 + with: + name: dist + path: dist/* + + - name: Upload docs + uses: actions/upload-artifact@v7 + with: + name: docs + path: docs/build/html/* + + build-docker: + needs: + - build-python + runs-on: ubuntu-latest + env: + VPU_DOCKER_REGISTRY: ${{ inputs.vpu_docker_registry }} + VPU_DOCKER_IMAGE_NAME: ${{ inputs.vpu_docker_image_name }} + VPU_DOCKER_REPOSITORIES: ${{ inputs.vpu_docker_repositories }} + COLUMNS: 160 + JFROG_CLI_BUILD_NAME: ${{ inputs.build_name }} + JFROG_CLI_BUILD_NUMBER: ${{ inputs.build_number }} + JFROG_CLI_TEMP_DIR: ${{ github.workspace }}/.jfrog-build" + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - uses: ./.github/actions/setup-jfrog + + - uses: astral-sh/setup-uv@v7 + + - name: Install VPU + run: uv tool install voraus-pipeline-utils + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Download python artifacts + uses: actions/download-artifact@v8 + with: + name: dist + path: dist + + - name: Build Docker Image(s) + run: vpu docker build -f Dockerfile.prod + + - name: Save Docker Image(s) + run: vpu docker save --output docker-images.tar + + - name: Upload docker images + uses: actions/upload-artifact@v7 + with: + name: docker-images + path: docker-images.tar + diff --git a/cookiecutter-pypackage/.github/workflows/lint.yml b/cookiecutter-pypackage/.github/workflows/lint.yml new file mode 100644 index 0000000..74c5721 --- /dev/null +++ b/cookiecutter-pypackage/.github/workflows/lint.yml @@ -0,0 +1,17 @@ +name: Linting + +on: [workflow_call] + +jobs: + lint-python: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - uses: astral-sh/setup-uv@v7 + + - name: Run static checks + run: uv run tox run -e lint diff --git a/cookiecutter-pypackage/.github/workflows/pipeline.yml b/cookiecutter-pypackage/.github/workflows/pipeline.yml new file mode 100644 index 0000000..bf9dfa2 --- /dev/null +++ b/cookiecutter-pypackage/.github/workflows/pipeline.yml @@ -0,0 +1,69 @@ +name: CI pipeline + +permissions: + contents: write + id-token: write + issues: write + pages: write + +on: + pull_request: + branches: + - main + push: + branches: + - "main" + tags: + - "*" + +jobs: + setup: + uses: ./.github/workflows/setup.yml + with: + package_name: 'cookiecutter-pypackage' + project: 'bypass' + scope: 'private' + maturity: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags') && 'production' || 'testing' }} + location: 'cloud' + registry: 'voraus.jfrog.io' + + build: + needs: + - setup + uses: ./.github/workflows/build.yml + secrets: inherit + with: + vpu_docker_registry: ${{ needs.setup.outputs.vpu_docker_registry }} + vpu_docker_image_name: ${{ needs.setup.outputs.vpu_docker_image_name }} + vpu_docker_repositories: ${{ needs.setup.outputs.vpu_docker_repositories }} + build_name: ${{ needs.setup.outputs.build_name }} + build_number: ${{ needs.setup.outputs.build_number }} + + tests-and-coverage: + uses: ./.github/workflows/tests_and_coverage.yml + needs: + - build + secrets: inherit + + lint: + uses: ./.github/workflows/lint.yml + needs: + - setup + secrets: inherit + + publish: + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + needs: + - setup + - build + - tests-and-coverage + - lint + uses: ./.github/workflows/publish.yml + secrets: inherit + with: + vpu_docker_registry: ${{ needs.setup.outputs.vpu_docker_registry }} + vpu_docker_image_name: ${{ needs.setup.outputs.vpu_docker_image_name }} + vpu_docker_repositories: ${{ needs.setup.outputs.vpu_docker_repositories }} + pypi_deploy_repo: ${{ needs.setup.outputs.pypi_deploy_repo }} + build_name: ${{ needs.setup.outputs.build_name }} + build_number: ${{ needs.setup.outputs.build_number }} diff --git a/cookiecutter-pypackage/.github/workflows/publish.yml b/cookiecutter-pypackage/.github/workflows/publish.yml new file mode 100644 index 0000000..688a7f7 --- /dev/null +++ b/cookiecutter-pypackage/.github/workflows/publish.yml @@ -0,0 +1,186 @@ +name: Publish + +on: + workflow_call: + inputs: + vpu_docker_registry: + required: true + type: string + vpu_docker_image_name: + required: true + type: string + vpu_docker_repositories: + required: true + type: string + pypi_deploy_repo: + required: true + type: string + build_name: + required: true + type: string + build_number: + required: true + type: string + +jobs: + publish-pypi: + runs-on: ubuntu-latest + env: + JFROG_CLI_BUILD_NAME: ${{ inputs.build_name }} + JFROG_CLI_BUILD_NUMBER: ${{ inputs.build_number }} + JFROG_CLI_TEMP_DIR: ${{ github.workspace }}/.jfrog-build + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - uses: ./.github/actions/setup-jfrog + + - name: Add Git info to build + run: jf rt build-add-git + + - uses: astral-sh/setup-uv@v7 + + - name: Install twine + run: uv tool install twine + + - name: Configure jf pip + run: jf pip-config --repo-resolve pypi --repo-deploy ${{ inputs.pypi_deploy_repo }} + + - name: Download python artifacts + uses: actions/download-artifact@v8 + with: + name: dist + path: dist + + - name: Publish PyPI + run: jf twine --module=voraus-pipeline-playground upload dist/* + + - name: Upload build info partials + if: success() + uses: actions/upload-artifact@v7 + with: + name: build-partials-${{ github.job }} + path: ${{ env.JFROG_CLI_TEMP_DIR }} + include-hidden-files: true + if-no-files-found: error + + publish-docker: + runs-on: ubuntu-latest + env: + VPU_DOCKER_REGISTRY: ${{ inputs.vpu_docker_registry }} + VPU_DOCKER_IMAGE_NAME: ${{ inputs.vpu_docker_image_name }} + VPU_DOCKER_REPOSITORIES: ${{ inputs.vpu_docker_repositories }} + COLUMNS: 160 + JFROG_CLI_BUILD_NAME: ${{ inputs.build_name }} + JFROG_CLI_BUILD_NUMBER: ${{ inputs.build_number }} + JFROG_CLI_TEMP_DIR: ${{ github.workspace }}/.jfrog-build" + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - uses: ./.github/actions/setup-jfrog + + - name: Add Git info to build + run: jf rt build-add-git + + - uses: astral-sh/setup-uv@v7 + + - name: Install VPU + run: uv tool install voraus-pipeline-utils + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Download all artifacts + uses: actions/download-artifact@v8 + with: + name: docker-images + + - name: Load docker images + run: docker load -i docker-images.tar + + - name: Publish Docker Image(s) + run: vpu docker push + + - name: Upload build info partials + if: success() + uses: actions/upload-artifact@v7 + with: + name: build-partials-${{ github.job }} + path: ${{ env.JFROG_CLI_TEMP_DIR }} + include-hidden-files: true + if-no-files-found: error + + publish-build-info: + needs: + - publish-pypi + - publish-docker + runs-on: ubuntu-latest + env: + JFROG_CLI_BUILD_NAME: ${{ inputs.build_name }} + JFROG_CLI_BUILD_NUMBER: ${{ inputs.build_number }} + JFROG_CLI_TEMP_DIR: ${{ github.workspace }}/.jfrog-build + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - uses: ./.github/actions/setup-jfrog + + - name: Add environment and git information to build info + run: | + jf rt build-collect-env + jf rt build-add-git + + - name: Download build info partials + uses: actions/download-artifact@v8 + with: + pattern: build-partials-* + path: ${{ env.JFROG_CLI_TEMP_DIR }} + merge-multiple: true + + - name: Publish unified build info + run: jf rt build-publish + + create-github-release: + runs-on: ubuntu-latest + needs: + - publish-pypi + - publish-docker + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Run JReleaser + uses: jreleaser/release-action@v2 + env: + JRELEASER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + JRELEASER_PROJECT_VERSION: ${{ github.ref_name }} + + publish-docs: + runs-on: ubuntu-latest + needs: + - publish-pypi + - publish-docker + steps: + - name: Download all artifacts + uses: actions/download-artifact@v8 + with: + name: docs + path: docs + + - name: Setup Pages + uses: actions/configure-pages@v5 + + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + with: + path: docs/ + + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/cookiecutter-pypackage/.github/workflows/setup.yml b/cookiecutter-pypackage/.github/workflows/setup.yml new file mode 100644 index 0000000..7b36c7d --- /dev/null +++ b/cookiecutter-pypackage/.github/workflows/setup.yml @@ -0,0 +1,73 @@ +name: Configuration + +on: + workflow_call: + inputs: + package_name: + required: true + type: string + project: + required: true + type: string + scope: + required: true + type: string + location: + required: true + type: string + maturity: + required: true + type: string + registry: + required: true + type: string + outputs: + vpu_docker_registry: + description: "Docker registry URL" + value: ${{ jobs.setup.outputs.vpu_docker_registry }} + vpu_docker_image_name: + description: "Docker image name" + value: ${{ jobs.setup.outputs.vpu_docker_image_name }} + vpu_docker_repositories: + description: "Docker repositories" + value: ${{ jobs.setup.outputs.vpu_docker_repositories }} + pypi_deploy_repo: + description: "PyPI deploy repo" + value: ${{ jobs.setup.outputs.pypi_deploy_repo }} + build_name: + description: "JFrog build name" + value: ${{ jobs.setup.outputs.build_name }} + build_number: + description: "JFrog build number" + value: ${{ jobs.setup.outputs.build_number }} + +jobs: + setup: + runs-on: ubuntu-latest + outputs: + vpu_docker_registry: ${{ steps.vars.outputs.vpu_docker_registry }} + vpu_docker_image_name: ${{ steps.vars.outputs.vpu_docker_image_name }} + vpu_docker_repositories: ${{ steps.vars.outputs.vpu_docker_repositories }} + pypi_deploy_repo: ${{ steps.vars.outputs.pypi_deploy_repo }} + build_name: ${{ steps.vars.outputs.build_name }} + build_number: ${{ steps.vars.outputs.build_number }} + steps: + - name: Set VPU variables + id: vars + run: | + VPU_DOCKER_IMAGE_NAME=${{ inputs.package_name }} + VPU_DOCKER_REGISTRY="${{ inputs.registry }}" + VPU_DOCKER_REPOSITORIES="${{ inputs.project }}-docker-${{ inputs.scope }}-${{ inputs.maturity }}-${{ inputs.location }}-local" + PYPI_DEPLOY_REPO="${{ inputs.project }}-pypi-${{ inputs.scope }}-${{ inputs.maturity }}-${{ inputs.location }}-local" + + # Unified build name and number for all stages + BUILD_NAME="${{ inputs.package_name }}" + BUILD_NUMBER="${{ github.run_number }}" + + # Output for other jobs + echo "vpu_docker_image_name=${VPU_DOCKER_IMAGE_NAME}" >> $GITHUB_OUTPUT + echo "vpu_docker_registry=${VPU_DOCKER_REGISTRY}" >> $GITHUB_OUTPUT + echo "vpu_docker_repositories=${VPU_DOCKER_REPOSITORIES}" >> $GITHUB_OUTPUT + echo "pypi_deploy_repo=${PYPI_DEPLOY_REPO}" >> $GITHUB_OUTPUT + echo "build_name=${BUILD_NAME}" >> $GITHUB_OUTPUT + echo "build_number=${BUILD_NUMBER}" >> $GITHUB_OUTPUT diff --git a/cookiecutter-pypackage/.github/workflows/tests_and_coverage.yml b/cookiecutter-pypackage/.github/workflows/tests_and_coverage.yml new file mode 100644 index 0000000..81d67c8 --- /dev/null +++ b/cookiecutter-pypackage/.github/workflows/tests_and_coverage.yml @@ -0,0 +1,28 @@ +name: Tests & Coverage + +on: [workflow_call] + + +jobs: + test-python: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [macos-latest, ubuntu-latest] + + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - uses: astral-sh/setup-uv@v7 + + - name: Run tests + run: uv run tox -f test + + - name: Upload combined python coverage + if: matrix.os == 'ubuntu-latest' + uses: actions/upload-artifact@v7 + with: + name: python-coverage-report + path: reports/coverage-python.xml diff --git a/cookiecutter-pypackage/.groovylintrc.json b/cookiecutter-pypackage/.groovylintrc.json deleted file mode 100644 index c6b6af8..0000000 --- a/cookiecutter-pypackage/.groovylintrc.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "extends": "recommended-jenkinsfile", - "rules": { - "DuplicateStringLiteral": { - "enabled": false - }, - "formatting.Indentation": { - "spacesPerIndentLevel": 2, - "severity": "warning" - } - } -} diff --git a/cookiecutter-pypackage/.jenkins/pod-spec.yml b/cookiecutter-pypackage/.jenkins/pod-spec.yml deleted file mode 100644 index 1187b34..0000000 --- a/cookiecutter-pypackage/.jenkins/pod-spec.yml +++ /dev/null @@ -1,17 +0,0 @@ -spec: - containers: - - name: build - image: artifactory.vorausrobotik.com/docker/voraus-build-image:8.1.0 - alwaysPullImage: true - securityContext: - runAsUser: 0 - command: - - cat - tty: true - volumeMounts: - - mountPath: /var/run/docker.sock - name: docker-sock - volumes: - - name: docker-sock - hostPath: - path: /var/run/docker.sock diff --git a/cookiecutter-pypackage/.jenkins/utils.groovy b/cookiecutter-pypackage/.jenkins/utils.groovy deleted file mode 100644 index 9801467..0000000 --- a/cookiecutter-pypackage/.jenkins/utils.groovy +++ /dev/null @@ -1,72 +0,0 @@ -/** - * Initializes JFrog CLI with the necessary configuration for the current build. - */ -void initJFrogCLI() { - withCredentials([usernamePassword( - credentialsId: 'jfrog-platform-credentials', - usernameVariable: 'USER', - passwordVariable: 'TOKEN' - )]) { - sh 'jf config add --user $USER --password $TOKEN --url ' + env.JFROG_PLATFORM_URL - } - sh "jf pip-config --repo-resolve pypi --repo-deploy ${env.PYPI_DEPLOY_REPO}" - String[] projectPathParts = env.JOB_NAME.split('/') - String jobName = URLDecoder.decode(projectPathParts[-2], 'utf-8') - String branchName = URLDecoder.decode(env.BRANCH_NAME, 'utf-8') - env.JFROG_CLI_BUILD_NAME = jobName - env.JFROG_CLI_BUILD_NUMBER = "${branchName}/${env.BUILD_NUMBER}" - env.JFROG_CLI_BUILD_URL = env.BUILD_URL - env.JFROG_CLI_REPORT_USAGE = false - env.JFROG_CLI_FAIL_NO_OP = true -} - -/** - * Publishes the build info to Artifactory and adds a summary and badge to the Jenkins build page. - */ -void publishBuildInfo() { - Map result = readJSON(text: sh(script: 'jf rt build-publish', returnStdout: true).trim()) - - String icon = 'symbol-cube plugin-ionicons-api' - String style = 'color: green' - String text = 'Artifactory Build Info' - String link = result['buildInfoUiUrl'] - - addSummary(icon: icon, style: style, text: text, link: link, target: '_blank') - addBadge(icon: icon, style: style, text: text, link: link, target: '_blank') -} - -/** - * Install all package dependencies so that they are linked in the build info JSON. - */ -void populateBuildInfo() { - // We need to use pip here because JFrog CLI doesn't support astral's uv yet - // More information: https://github.com/jfrog/jfrog-cli-artifactory/issues/212 - sh 'python3 -m venv venvDependencies' - sh '''\ - . venvDependencies/bin/activate && \ - jf pip install \ - --module=voraus-example-application \ - --no-cache-dir \ - --force-reinstall \ - . - ''' -} - -void publishGitHubRelease() { - withCredentials([usernamePassword( - credentialsId: 'github-app', - usernameVariable: 'GITHUB_APP', - passwordVariable: 'GITHUB_ACCESS_TOKEN' - )]) { - withEnv([ - "JRELEASER_PROJECT_NAME=${env.PACKAGE_NAME}", - "JRELEASER_PROJECT_VERSION=${env.TAG_NAME}", - "JRELEASER_GITHUB_TOKEN=${env.GITHUB_ACCESS_TOKEN}".toString(), - 'JRELEASER_SKIP_TAG=true' - ]) { - sh 'jreleaser release' - } - } -} - -return this diff --git a/cookiecutter-pypackage/Jenkinsfile b/cookiecutter-pypackage/Jenkinsfile deleted file mode 100644 index 1c6deca..0000000 --- a/cookiecutter-pypackage/Jenkinsfile +++ /dev/null @@ -1,148 +0,0 @@ -// Pipeline utilities loaded from .jenkins/utils.groovy in the preparation stage -def utils = null - -pipeline { - agent { - kubernetes { - defaultContainer 'build' - yamlFile 'cookiecutter-pypackage/.jenkins/pod-spec.yml' - } - } - environment { - PACKAGE_NAME = 'cookiecutter-pypackage' - PROJECT = 'default' - SCOPE = 'private' - LOCATION = 'onprem' - VPU_DOCKER_REGISTRY = 'artifactory.vorausrobotik.com' - COLUMNS = '160' - } - stages { - stage('Prepare') { - parallel { - stage('Init build environment') { - steps { - dir('cookiecutter-pypackage/') { - sh 'uv sync --frozen'} - } - } - stage('Init JFrog CLI') { - steps { - dir('cookiecutter-pypackage/') { - script { - env.MATURITY = env.TAG_NAME != null ? 'production' : 'testing' - env.PYPI_DEPLOY_REPO = "${env.PROJECT}-pypi-${env.SCOPE}-${env.MATURITY}-onprem-local" - utils = load '.jenkins/utils.groovy' - utils.initJFrogCLI() - }} - } - } - } - } - stage('Build Package') { - steps { - dir('cookiecutter-pypackage/') { - sh 'uv run tox run -e build'} - } - } - stage('Protect Wheel') { - steps { - dir('cookiecutter-pypackage/') { - withCredentials([ - usernamePassword( - credentialsId: 'voraus-software-protector-api-key', - usernameVariable: 'PROTECTOR_ACCESS_KEY_ID', - passwordVariable: 'PROTECTOR_ACCESS_SECRET_KEY' - ) - ]) { - sh 'vpu protect wheel dist/*.whl --identifier=cookiecutter-pypackage-1' - }} - } - } - stage('Build Docker') { - steps { - dir('cookiecutter-pypackage/') { - sh 'uv run tox run -e build-docker'} - } - } - stage('Parallel') { - parallel { - stage('Test') { - steps { - dir('cookiecutter-pypackage/') { - // Initialize the test environments in parallel - sh 'uv run tox run-parallel -f test --notest --parallel-no-spinner --parallel-live' - // Run the test environments in sequence in the precommissioned environments - sh 'uv run tox run-parallel -f test --skip-pkg-install -- --maxfail 10'} - } - } - stage('Docs') { - steps { - dir('cookiecutter-pypackage/') { - sh 'uv run tox run -e docs'} - } - } - stage('Lint') { - steps { - dir('cookiecutter-pypackage/') { - sh 'uv run tox run -e lint'} - } - } - stage('Populate build info') { - steps { - dir('cookiecutter-pypackage/') { - script { - utils.populateBuildInfo() - }} - } - } - } - } - stage('Publish') { - steps { - dir('cookiecutter-pypackage/') { - sh 'uv run tox run -e publish' - script { - utils.publishBuildInfo() - }} - } - } - stage('Publish GitHub Release') { - when { - buildingTag() - } - steps { - dir('cookiecutter-pypackage/') { - script { - utils.publishGitHubRelease() - }} - } - } - } - post { - always { - dir('cookiecutter-pypackage/') { - junit 'reports/pytest.xml' - recordCoverage( - tools: [ - [ - parser: 'COBERTURA', - pattern: 'reports/coverage.xml' - ] - ] - ) - archiveArtifacts artifacts: 'dist/**', allowEmptyArchive: true} - } - success { - dir('cookiecutter-pypackage/') { - publishHTML([ - allowMissing: false, - alwaysLinkToLastBuild: false, - keepAll: false, - reportDir: 'docs/build/html/', - reportFiles: 'index.html', - reportName: 'Documentation', - reportTitles: '' - ])} - } - } -} diff --git a/test_config.yaml b/test_config.yaml index 22ebe9e..a625038 100644 --- a/test_config.yaml +++ b/test_config.yaml @@ -8,7 +8,7 @@ default_context: import_name: "cookiecutter_pypackage" project_short_description: "A cookiecutter for Python packages." url: "https://www.example.com" - ci_tool: "jenkins" + ci_tool: "github_actions" use_github_pr_lint: "True" use_github_dependabot: "True" protect_wheel: "True"