diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0344082..e379aea 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -80,12 +80,6 @@ jobs: node-version: 20 cache: "npm" - - name: Install FlatBuffers compiler - run: | - curl -L https://github.com/google/flatbuffers/releases/download/v25.2.10/Linux.flatc.binary.clang++-18.zip -o flatc.zip - unzip flatc.zip -d /usr/local/bin/ - chmod +x /usr/local/bin/flatc - - name: Install dependencies run: npm ci diff --git a/.gitignore b/.gitignore index 0da9b78..79b4148 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,9 @@ # Dependencies node_modules/ +# Local tooling (downloaded by scripts/ensure-flatc.sh) +.bin/ + # Build output dist/ diff --git a/README.md b/README.md index 73d7367..9230c11 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,10 @@ npm run test npm run typecheck ``` +To regenerate FlatBuffers TypeScript after syncing schemas, run `npm run generate:fbs`. +The pinned `flatc` compiler is downloaded automatically if not already available. +The version is set in `scripts/ensure-flatc.sh`. + ## API ### IcechunkStore diff --git a/package.json b/package.json index ba92d9d..a5ea00a 100644 --- a/package.json +++ b/package.json @@ -34,8 +34,8 @@ "prepare": "test -d tests/data || ./scripts/sync-fixtures.sh && test -d flatbuffers || ./scripts/sync-flatbuffers.sh", "sync:fixtures": "./scripts/sync-fixtures.sh", "sync:fbs": "./scripts/sync-flatbuffers.sh", - "generate:fbs": "flatc -T -o ./src/format/flatbuffers --gen-all ./flatbuffers/all.fbs && node scripts/postgen-fbs.cjs ./src/format/flatbuffers && prettier --write src/format/flatbuffers/generated", - "check:fbs": "flatc -T -o /tmp/fbs-check --gen-all ./flatbuffers/all.fbs && node scripts/postgen-fbs.cjs /tmp/fbs-check && prettier --write /tmp/fbs-check/generated && diff -r /tmp/fbs-check/generated src/format/flatbuffers/generated" + "generate:fbs": "FLATC=\"$(./scripts/ensure-flatc.sh)\" && \"$FLATC\" -T -o ./src/format/flatbuffers --gen-all ./flatbuffers/all.fbs && node scripts/postgen-fbs.cjs ./src/format/flatbuffers && prettier --write src/format/flatbuffers/generated", + "check:fbs": "rm -rf /tmp/fbs-check && FLATC=\"$(./scripts/ensure-flatc.sh)\" && \"$FLATC\" -T -o /tmp/fbs-check --gen-all ./flatbuffers/all.fbs && node scripts/postgen-fbs.cjs /tmp/fbs-check && prettier --write /tmp/fbs-check/generated && diff -r /tmp/fbs-check/generated src/format/flatbuffers/generated" }, "keywords": [ "icechunk", diff --git a/scripts/ensure-flatc.sh b/scripts/ensure-flatc.sh new file mode 100755 index 0000000..f8503cd --- /dev/null +++ b/scripts/ensure-flatc.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Ensure the pinned flatc version is available at .bin/flatc. +# Downloads it if missing or wrong version. Prints the path on stdout. +# +# Usage: +# FLATC="$(./scripts/ensure-flatc.sh)" +# "$FLATC" -T -o ./src/format/flatbuffers --gen-all ./flatbuffers/all.fbs + +set -euo pipefail + +FLATC_VERSION="25.12.19" + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_DIR="$(dirname "$SCRIPT_DIR")" +BIN_DIR="$PROJECT_DIR/.bin" +FLATC="$BIN_DIR/flatc" + +# Check if we already have the right version +if [ -x "$FLATC" ]; then + current=$("$FLATC" --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' || echo "") + if [ "$current" = "$FLATC_VERSION" ]; then + echo "$FLATC" + exit 0 + fi + echo "flatc in .bin/ is v$current, need v$FLATC_VERSION" >&2 +fi + +# Download +case "$(uname -s)" in + Linux) ASSET="Linux.flatc.binary.clang++-18.zip" ;; + Darwin) ASSET="Mac.flatc.binary.zip" ;; + *) + echo "error: cannot download flatc for $(uname -s)" >&2 + echo "Install flatc v$FLATC_VERSION to .bin/flatc manually." >&2 + exit 1 + ;; +esac + +mkdir -p "$BIN_DIR" + +URL="https://github.com/google/flatbuffers/releases/download/v${FLATC_VERSION}/${ASSET}" +TMPDIR=$(mktemp -d) +trap 'rm -rf "$TMPDIR"' EXIT + +echo "Downloading flatc v${FLATC_VERSION}..." >&2 +curl -fsSL "$URL" -o "$TMPDIR/flatc.zip" +unzip -o -q "$TMPDIR/flatc.zip" -d "$BIN_DIR/" +chmod +x "$FLATC" + +echo "$FLATC" diff --git a/src/format/flatbuffers/generated/move-operation.ts b/src/format/flatbuffers/generated/move-operation.ts index 4506a20..e2a6b30 100644 --- a/src/format/flatbuffers/generated/move-operation.ts +++ b/src/format/flatbuffers/generated/move-operation.ts @@ -2,6 +2,9 @@ import * as flatbuffers from "flatbuffers"; +import { NodeType } from "../generated/node-type.js"; +import { ObjectId8 } from "../generated/object-id8.js"; + export class MoveOperation { bb: flatbuffers.ByteBuffer | null = null; bb_pos = 0; @@ -38,4 +41,16 @@ export class MoveOperation { ? this.bb!.__string(this.bb_pos + offset, optionalEncoding) : null; } + + nodeId(obj?: ObjectId8): ObjectId8 | null { + const offset = this.bb!.__offset(this.bb_pos, 8); + return offset + ? (obj || new ObjectId8()).__init(this.bb_pos + offset, this.bb!) + : null; + } + + nodeType(): NodeType { + const offset = this.bb!.__offset(this.bb_pos, 10); + return offset ? this.bb!.readUint8(this.bb_pos + offset) : NodeType.Group; + } } diff --git a/src/format/flatbuffers/generated/node-type.ts b/src/format/flatbuffers/generated/node-type.ts new file mode 100644 index 0000000..43122d2 --- /dev/null +++ b/src/format/flatbuffers/generated/node-type.ts @@ -0,0 +1,6 @@ +// automatically generated by the FlatBuffers compiler, do not modify + +export enum NodeType { + Group = 0, + Array = 1, +}