chore(IDX): split out aws creds

.github/actions/upload-artifacts/action.yaml

@@ -2,8 +2,6 @@ name: Upload Artifacts
 description: |
   Run an artifact upload command and set a step summary. This expects the upload command to write one line per artifact to stdout.
 inputs:
-  credentials-content:
-    required: true
   upload-command:
     required: true
   name:
@@ -12,16 +10,6 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - name: Write AWS credentials
-      shell: bash
-      run: |
-        credentials_dir="${HOME}/.aws/credentials"
-        if ! [ -e "$credentials_dir" ]; then
-          echo "writing credentials"
-          mkdir -p "$(dirname "${credentials_dir}")"
-          echo '${{ inputs.credentials-content }}' >"$credentials_dir"
-        fi
-
     - name: Upload to S3
       uses: ./.github/actions/bazel
       with:

.github/workflows/ci-main.yml

@@ -150,7 +150,7 @@ jobs:
     needs: [config]
     environment:
       # The `Upload artifacts` step below only executes if `release-build == 'true'` which is the case on protected branches.
-      # It also requires the CLOUD_CREDENTIALS_CONTENT secret which is only available in the `upload-artifacts` environment.
+      # It also requires the AWS and CF secrets which are only available in the `upload-artifacts` environment.
       # This environment is also only active on protected branches. That's why we require the `upload-artifacts` environment
       # when `release-build == 'true'` but not otherwise.
       name: ${{ needs.config.outputs.release-build == 'true' && 'upload-artifacts' || ''}}
@@ -249,20 +249,24 @@ jobs:
       - name: Upload artifacts
         uses: ./.github/actions/upload-artifacts
         if: needs.config.outputs.release-build == 'true'
+        env:
+          AWS_AWS_ACCESS_KEY_ID: ${{ secrets.AWS_AWS_ACCESS_KEY_ID }}
+          AWS_AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_AWS_SECRET_ACCESS_KEY }}
+          CF_AWS_ACCESS_KEY_ID: ${{ secrets.CF_AWS_ACCESS_KEY_ID }}
+          CF_AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_AWS_SECRET_ACCESS_KEY }}
         with:
           name: Bazel Test All
           # with --check_up_to_date Bazel will error out if the artifacts
           # to be uploaded were not built in the build step above
           # (this ensures that the exact artifacts built above are uploaded)
           upload-command: bazel run --check_up_to_date //:upload-artifacts
-          credentials-content: ${{ secrets.CLOUD_CREDENTIALS_CONTENT }}
 
   bazel-test-macos-intel:
     name: Bazel Test macOS Intel
     needs: [config]
     environment:
       # The `Upload artifacts` step below only executes if `release-build == 'true'` which is the case on protected branches.
-      # It also requires the CLOUD_CREDENTIALS_CONTENT secret which is only available in the `upload-artifacts` environment.
+      # It also requires the AWS & CF secrets which are only available in the `upload-artifacts` environment.
       # This environment is also only active on protected branches. That's why we require the `upload-artifacts` environment
       # when `release-build == 'true'` but not otherwise.
       name: ${{ needs.config.outputs.release-build == 'true' && 'upload-artifacts' || ''}}
@@ -306,10 +310,14 @@ jobs:
         # NOTE: GHA output quirk, 'true' is a string
         if: ${{ needs.config.outputs.full_macos_build == 'true' && needs.config.outputs.release-build == 'true' }}
         uses: ./.github/actions/upload-artifacts
+        env:
+          AWS_AWS_ACCESS_KEY_ID: ${{ secrets.AWS_AWS_ACCESS_KEY_ID }}
+          AWS_AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_AWS_SECRET_ACCESS_KEY }}
+          CF_AWS_ACCESS_KEY_ID: ${{ secrets.CF_AWS_ACCESS_KEY_ID }}
+          CF_AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_AWS_SECRET_ACCESS_KEY }}
         with:
           name: macOS Intel artifacts
           upload-command: bazel run --check_up_to_date //:upload-artifacts
-          credentials-content: ${{ secrets.CLOUD_CREDENTIALS_CONTENT }}
 
       - name: Purge Bazel Output
         if: always()
@@ -407,8 +415,8 @@ jobs:
     name: Upload external artifacts
     needs: [bazel-test-arm64, config]
     if: ${{ needs.config.outputs.release-build == 'true' }} # GHA output quirk, 'true' is a string
-    # The `Upload artifacts` step below requires the CLOUD_CREDENTIALS_CONTENT secret
-    # which is only available in the `upload-artifacts` environment.
+    # The `Upload artifacts` step below requires the AWS & CF secrets
+    # which are only available in the `upload-artifacts` environment.
     environment: upload-artifacts
     runs-on: *dind-large-setup
     container: *container-setup
@@ -445,10 +453,14 @@ jobs:
 
       - name: Upload artifacts
         uses: ./.github/actions/upload-artifacts
+        env:
+          AWS_AWS_ACCESS_KEY_ID: ${{ secrets.AWS_AWS_ACCESS_KEY_ID }}
+          AWS_AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_AWS_SECRET_ACCESS_KEY }}
+          CF_AWS_ACCESS_KEY_ID: ${{ secrets.CF_AWS_ACCESS_KEY_ID }}
+          CF_AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_AWS_SECRET_ACCESS_KEY }}
         with:
           name: arm64 artifacts
           upload-command: bazel run --config=stamped //:artifact-uploader -- ${{ steps.prepare-bundle.outputs.bundledir }}
-          credentials-content: ${{ secrets.CLOUD_CREDENTIALS_CONTENT }}
 
   python-ci-tests:
     name: Python CI Tests

ci/src/artifacts/upload.sh

@@ -41,7 +41,9 @@ upload() {
     # NOTE: we upload a "directory" and narrow down the upload with --files-from so that only
     # the current artifact is uploaded. Without this, --immutable does not work as expected.
     # https://github.com/rclone/rclone/issues/4921
-    AWS_PROFILE=default rclone \
+    AWS_ACCESS_KEY_ID="${AWS_AWS_ACCESS_KEY_ID:?AWS credentials required for AWS upload}" \
+        AWS_SECRET_ACCESS_KEY="${AWS_AWS_SECRET_ACCESS_KEY:?AWS credentials required for AWS upload}" \
+        rclone \
         "${rclone_common_flags[@]}" \
         --s3-provider=AWS \
         --s3-region=eu-central-1 \
@@ -55,13 +57,13 @@ upload() {
     log "done uploading to AWS"
 
     # Upload to Cloudflare's R2 (S3)
-    # using profile 'cf' to look up the right creds in ~/.aws/credentials
     log "uploading to Cloudflare"
-    # unset env vars so it doesn't use AWS credentials
-    env -u AWS_ACCESS_KEY_ID -u AWS_SECRET_ACCESS_KEY -u AWS_SESSION_TOKEN
-    AWS_PROFILE=cf rclone -v \
+    AWS_ACCESS_KEY_ID="${CF_AWS_ACCESS_KEY_ID:?CloudFlare R2 credentials required for CF upload}" \
+        AWS_SECRET_ACCESS_KEY="${CF_AWS_SECRET_ACCESS_KEY:?Cloudflare R2 credentials required for CF upload}" \
+        rclone -v \
         "${rclone_common_flags[@]}" \
         --s3-provider=Cloudflare \
+        --s3-region=weur \
         --s3-endpoint=https://64059940cc95339fc7e5888f431876ee.r2.cloudflarestorage.com \
         --s3-env-auth \
         copy \