From 1204cb343b24f8d272c638b076d70e94a2456e62 Mon Sep 17 00:00:00 2001 From: Dragan Milic Date: Thu, 12 Feb 2026 16:12:23 +0100 Subject: [PATCH 1/5] proposal for bitemporality --- BITEMPORALITY-PROPOSAL.md | 97 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 BITEMPORALITY-PROPOSAL.md diff --git a/BITEMPORALITY-PROPOSAL.md b/BITEMPORALITY-PROPOSAL.md new file mode 100644 index 0000000..eb551e1 --- /dev/null +++ b/BITEMPORALITY-PROPOSAL.md @@ -0,0 +1,97 @@ +# Proposal to Add Bitemporality + +## Context + +The current implementation does not support bitemporality. This causes two problems: + +* Paginated queries cannot guarantee stable results across multiple requests, since the underlying data may change between pages. +* Archive nodes cannot query historical data at a specific block height. + +The root cause is the nature of bitmap indexes: each bitmap contains a summary of all entities that currently match a given key/value pair. Once computed for the current state, there is no way to reconstruct what the bitmap looked like at an earlier block. + +## Proposal + +We propose a video-compression-inspired approach to allow querying at any block height. + +The core idea is to maintain two types of bitmap entries: **keyframes** (full bitmaps representing the complete state) and **delta frames** (XOR diffs against the previous frame). Applying a delta frame to the previous state reconstructs the bitmap at that point in time. + +This approach is efficient because the delta frames are typically much smaller than the full bitmap. To bound the cost of reconstruction, we limit the number of consecutive delta frames to a constant `C` (e.g. 128). After `C` deltas, a new keyframe is stored. + +``` +Block: 10 11 12 ... 138 139 + ┌───────┐ ┌───────┐ ┌───────┐ ┌───────┐ ┌───────┐ +Type: │ KEY │ │ DELTA │ │ DELTA │ ... │ DELTA │ │ KEY │ + └───────┘ └───────┘ └───────┘ └───────┘ └───────┘ + │ │ │ │ │ + │ │ │ │ │ + ▼ ▼ ▼ ▼ ▼ + Full XOR with XOR with XOR with Full + bitmap block 10 block 11 block 137 bitmap + state state state + +To reconstruct state at block 12: + state(10) XOR delta(11) XOR delta(12) + +To reconstruct state at block 139: + Just read the keyframe directly +``` + +### Storing Bitmaps + +When storing a bitmap for a given key/value pair at a new block: + +1. If no bitmap exists for this key/value pair, the previous state is assumed to be an empty bitmap. The new bitmap is stored as a keyframe. +2. If existing bitmap entries exist (a keyframe and optionally delta frames), they are loaded and the XOR chain is applied to reconstruct the current state. +3. The new bitmap is produced by applying the incoming changes to the reconstructed current state. +4. If the number of delta frames since the last keyframe is less than `C`, the XOR of the old and new bitmaps is stored as a new delta frame. +5. If the number of delta frames has reached `C`, the new bitmap is stored as a fresh keyframe instead. + +### Reading Bitmaps + +When reading a bitmap for a given key/value pair at a target block `B`: + +1. Find the most recent keyframe at or before block `B`. +2. Load all delta frames between that keyframe and block `B` (inclusive). +3. Starting from the keyframe, apply each delta frame in sequence using XOR to reconstruct the bitmap state at block `B`. + +If block `B` coincides with a keyframe, no delta application is needed and the keyframe is returned directly. In the worst case, up to `C` XOR operations are required. + +## Changes Needed to the Current Schema + +- The `payloads` table gains `from_block` and `to_block` columns to track which block range each payload version is valid for. +- Each bitmap table (`string_attributes_values_bitmaps`, `numeric_attributes_values_bitmaps`) gains two columns: + - `block` — the block height at which this bitmap entry was recorded. + - `is_full_bitmap` — a boolean flag indicating whether the entry is a keyframe (`true`) or a delta frame (`false`). + +## Pruning Old State + +To prune all state before a block `B`: + +1. Delete all payloads where `to_block <= B`. +2. Find the most recent keyframe at or before block `B` for each key/value pair. +3. Delete all bitmap entries (keyframes and deltas) that precede that keyframe. + +The keyframe itself must be kept, as it serves as the base for any subsequent delta frames. + +## Support for Reorgs + +In the event of a chain reorganization that invalidates all blocks after block `B`: + +1. Delete all payloads where `to_block > B`. +2. Delete all bitmap entries where `block > B`. + +No reconstruction is needed. The remaining keyframes and delta frames still form a valid chain up to block `B`. + +## Consequences + +### Query Performance + +Querying becomes slower because reading a bitmap now requires reconstructing state from a keyframe and up to `C` delta frames. The performance overhead is directly driven by the choice of `C`. A smaller `C` reduces reconstruction cost but increases storage due to more frequent keyframes. + +### Increased Storage + +Since delta frames are stored alongside keyframes, the database stores more data than the current single-bitmap-per-key approach. The actual impact depends on the rate of change between blocks and is difficult to predict analytically. + +### Increased Block Processing Time + +Each new block requires computing and storing a delta (or a new keyframe), which adds overhead compared to the current approach of overwriting the bitmap in place. The real impact should be measured empirically with both synthetic and production data. From b0fd96663c493c34d189eb610d4a3844cf114e99 Mon Sep 17 00:00:00 2001 From: Dragan Milic Date: Tue, 10 Mar 2026 13:13:23 +0100 Subject: [PATCH 2/5] implement bitemporality --- .claude/settings.local.json | 4 +- bitmap_cache.go | 199 ++++--- maintenance.go | 71 +++ pusher/push_iterator_test.go | 26 +- query/evaluate.go | 384 ++++++-------- query_rpc.go | 1 + sqlitestore.go | 156 +++--- sqlitestore_test.go | 131 ++--- store/bitmap_chain.go | 212 ++++++++ store/db.go | 584 +++++++++++++-------- store/evals.sql.go | 474 +++++++++-------- store/models.go | 18 +- store/querier.go | 77 ++- store/queries.sql.go | 527 +++++++++++++++---- store/queries/evals.sql | 172 +++--- store/queries/queries.sql | 156 ++++-- store/queries/retrieve.sql | 12 +- store/retrieve.sql.go | 60 ++- store/schema/000002_bitemporality.down.sql | 64 +++ store/schema/000002_bitemporality.up.sql | 60 +++ store/sqlc.yaml | 16 +- update_index.go | 52 ++ 22 files changed, 2301 insertions(+), 1155 deletions(-) create mode 100644 maintenance.go create mode 100644 store/bitmap_chain.go create mode 100644 store/schema/000002_bitemporality.down.sql create mode 100644 store/schema/000002_bitemporality.up.sql create mode 100644 update_index.go diff --git a/.claude/settings.local.json b/.claude/settings.local.json index baaa72b..afc5737 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -9,7 +9,9 @@ "Bash(go get:*)", "Bash(go test:*)", "Bash(go mod tidy:*)", - "Bash(mkdir:*)" + "Bash(mkdir:*)", + "Bash(grep:*)", + "Bash(go vet:*)" ] } } diff --git a/bitmap_cache.go b/bitmap_cache.go index 13ed061..12f7735 100644 --- a/bitmap_cache.go +++ b/bitmap_cache.go @@ -2,7 +2,6 @@ package sqlitebitmapstore import ( "context" - "database/sql" "fmt" "runtime" @@ -16,136 +15,120 @@ type nameValue[T any] struct { } type bitmapCache struct { - st store.Querier + st *store.Queries + block uint64 stringBitmaps map[nameValue[string]]*store.Bitmap numericBitmaps map[nameValue[uint64]]*store.Bitmap + + // Old bitmaps (state before modifications) for computing deltas. + stringOldBitmaps map[nameValue[string]]*store.Bitmap + numericOldBitmaps map[nameValue[uint64]]*store.Bitmap } -func newBitmapCache(st store.Querier) *bitmapCache { +func newBitmapCache(st *store.Queries, block uint64) *bitmapCache { return &bitmapCache{ - st: st, - stringBitmaps: make(map[nameValue[string]]*store.Bitmap), - numericBitmaps: make(map[nameValue[uint64]]*store.Bitmap), + st: st, + block: block, + stringBitmaps: make(map[nameValue[string]]*store.Bitmap), + numericBitmaps: make(map[nameValue[uint64]]*store.Bitmap), + stringOldBitmaps: make(map[nameValue[string]]*store.Bitmap), + numericOldBitmaps: make(map[nameValue[uint64]]*store.Bitmap), + } +} + +func cloneBitmap(bm *store.Bitmap) *store.Bitmap { + if bm == nil || bm.Bitmap == nil { + return store.NewBitmap() } + return &store.Bitmap{Bitmap: bm.Bitmap.Clone()} } func (c *bitmapCache) AddToStringBitmap(ctx context.Context, name string, value string, id uint64) (err error) { k := nameValue[string]{name: name, value: value} bitmap, ok := c.stringBitmaps[k] if !ok { - bitmap, err = c.st.GetStringAttributeValueBitmap(ctx, store.GetStringAttributeValueBitmapParams{Name: name, Value: value}) - - if err != nil && err != sql.ErrNoRows { - return fmt.Errorf("failed to get string attribute %q value %q bitmap: %w", name, value, err) - } - - if bitmap == nil { - bitmap = store.NewBitmap() + bitmap, err = c.st.ReconstructLatestStringBitmap(ctx, name, value) + if err != nil { + return fmt.Errorf("failed to reconstruct string bitmap %q=%q: %w", name, value, err) } - + c.stringOldBitmaps[k] = cloneBitmap(bitmap) c.stringBitmaps[k] = bitmap } bitmap.Add(id) - return nil - } func (c *bitmapCache) RemoveFromStringBitmap(ctx context.Context, name string, value string, id uint64) (err error) { k := nameValue[string]{name: name, value: value} bitmap, ok := c.stringBitmaps[k] if !ok { - bitmap, err = c.st.GetStringAttributeValueBitmap(ctx, store.GetStringAttributeValueBitmapParams{Name: name, Value: value}) - - if err != nil && err != sql.ErrNoRows { - return fmt.Errorf("failed to get string attribute %q value %q bitmap: %w", name, value, err) - } - - if bitmap == nil { - bitmap = store.NewBitmap() + bitmap, err = c.st.ReconstructLatestStringBitmap(ctx, name, value) + if err != nil { + return fmt.Errorf("failed to reconstruct string bitmap %q=%q: %w", name, value, err) } - + c.stringOldBitmaps[k] = cloneBitmap(bitmap) c.stringBitmaps[k] = bitmap } bitmap.Remove(id) - return nil - } func (c *bitmapCache) AddToNumericBitmap(ctx context.Context, name string, value uint64, id uint64) (err error) { k := nameValue[uint64]{name: name, value: value} bitmap, ok := c.numericBitmaps[k] if !ok { - bitmap, err = c.st.GetNumericAttributeValueBitmap(ctx, store.GetNumericAttributeValueBitmapParams{Name: name, Value: value}) - - if err != nil && err != sql.ErrNoRows { - return fmt.Errorf("failed to get numeric attribute %q value %q bitmap: %w", name, value, err) - } - - if bitmap == nil { - bitmap = store.NewBitmap() + bitmap, err = c.st.ReconstructLatestNumericBitmap(ctx, name, value) + if err != nil { + return fmt.Errorf("failed to reconstruct numeric bitmap %q=%d: %w", name, value, err) } - + c.numericOldBitmaps[k] = cloneBitmap(bitmap) c.numericBitmaps[k] = bitmap } bitmap.Add(id) - return nil - } func (c *bitmapCache) RemoveFromNumericBitmap(ctx context.Context, name string, value uint64, id uint64) (err error) { k := nameValue[uint64]{name: name, value: value} bitmap, ok := c.numericBitmaps[k] if !ok { - bitmap, err = c.st.GetNumericAttributeValueBitmap(ctx, store.GetNumericAttributeValueBitmapParams{Name: name, Value: value}) - - if err != nil && err != sql.ErrNoRows { - return fmt.Errorf("failed to get numeric attribute %q value %q bitmap: %w", name, value, err) - } - - if bitmap == nil { - bitmap = store.NewBitmap() + bitmap, err = c.st.ReconstructLatestNumericBitmap(ctx, name, value) + if err != nil { + return fmt.Errorf("failed to reconstruct numeric bitmap %q=%d: %w", name, value, err) } - + c.numericOldBitmaps[k] = cloneBitmap(bitmap) c.numericBitmaps[k] = bitmap } bitmap.Remove(id) - return nil - } func (c *bitmapCache) Flush(ctx context.Context) (err error) { - + // RunOptimize all bitmaps in parallel (CPU-bound). eg := &errgroup.Group{} - eg.SetLimit(runtime.NumCPU()) for _, bitmap := range c.stringBitmaps { - if bitmap.IsEmpty() { - continue + if !bitmap.IsEmpty() { + eg.Go(func() error { + bitmap.RunOptimize() + return nil + }) } - eg.Go(func() error { - bitmap.RunOptimize() - return nil - }) } for _, bitmap := range c.numericBitmaps { - if bitmap.IsEmpty() { - continue + if !bitmap.IsEmpty() { + eg.Go(func() error { + bitmap.RunOptimize() + return nil + }) } - eg.Go(func() error { - bitmap.RunOptimize() - return nil - }) } err = eg.Wait() @@ -153,36 +136,92 @@ func (c *bitmapCache) Flush(ctx context.Context) (err error) { return fmt.Errorf("failed to run optimize: %w", err) } - for k, bitmap := range c.stringBitmaps { + // Write string bitmaps as keyframes or deltas. + for k, newBitmap := range c.stringBitmaps { + oldBitmap := c.stringOldBitmaps[k] + if oldBitmap == nil { + oldBitmap = store.NewBitmap() + } - if bitmap.IsEmpty() { - err = c.st.DeleteStringAttributeValueBitmap(ctx, store.DeleteStringAttributeValueBitmapParams{Name: k.name, Value: k.value}) - if err != nil { - return fmt.Errorf("failed to delete string attribute %q value %q bitmap: %w", k.name, k.value, err) - } + // Skip if bitmap hasn't changed. + if newBitmap.Bitmap.Equals(oldBitmap.Bitmap) { continue } - err = c.st.UpsertStringAttributeValueBitmap(ctx, store.UpsertStringAttributeValueBitmapParams{Name: k.name, Value: k.value, Bitmap: bitmap}) + deltaCount, err := c.st.CountDeltasSinceLastKeyframeString(ctx, store.CountDeltasSinceLastKeyframeStringParams{ + Name: k.name, Value: k.value, + }) + if err != nil { + return fmt.Errorf("failed to count deltas for string %q=%q: %w", k.name, k.value, err) + } + + isKeyframe := deltaCount >= store.KeyframeInterval || oldBitmap.IsEmpty() + + if isKeyframe { + err = c.st.InsertStringBitmapEntry(ctx, store.InsertStringBitmapEntryParams{ + Name: k.name, + Value: k.value, + Block: c.block, + IsFullBitmap: true, + Bitmap: newBitmap, + }) + } else { + delta := store.ComputeDelta(oldBitmap.Bitmap, newBitmap.Bitmap) + err = c.st.InsertStringBitmapEntry(ctx, store.InsertStringBitmapEntryParams{ + Name: k.name, + Value: k.value, + Block: c.block, + IsFullBitmap: false, + Bitmap: delta, + }) + } if err != nil { - return fmt.Errorf("failed to upsert string attribute %q value %q bitmap: %w", k.name, k.value, err) + return fmt.Errorf("failed to insert string bitmap entry %q=%q: %w", k.name, k.value, err) } } - for k, bitmap := range c.numericBitmaps { + // Write numeric bitmaps as keyframes or deltas. + for k, newBitmap := range c.numericBitmaps { + oldBitmap := c.numericOldBitmaps[k] + if oldBitmap == nil { + oldBitmap = store.NewBitmap() + } - if bitmap.IsEmpty() { - err = c.st.DeleteNumericAttributeValueBitmap(ctx, store.DeleteNumericAttributeValueBitmapParams{Name: k.name, Value: k.value}) - if err != nil { - return fmt.Errorf("failed to delete numeric attribute %q value %q bitmap: %w", k.name, k.value, err) - } + if newBitmap.Bitmap.Equals(oldBitmap.Bitmap) { continue } - err = c.st.UpsertNumericAttributeValueBitmap(ctx, store.UpsertNumericAttributeValueBitmapParams{Name: k.name, Value: k.value, Bitmap: bitmap}) + deltaCount, err := c.st.CountDeltasSinceLastKeyframeNumeric(ctx, store.CountDeltasSinceLastKeyframeNumericParams{ + Name: k.name, Value: k.value, + }) + if err != nil { + return fmt.Errorf("failed to count deltas for numeric %q=%d: %w", k.name, k.value, err) + } + + isKeyframe := deltaCount >= store.KeyframeInterval || oldBitmap.IsEmpty() + + if isKeyframe { + err = c.st.InsertNumericBitmapEntry(ctx, store.InsertNumericBitmapEntryParams{ + Name: k.name, + Value: k.value, + Block: c.block, + IsFullBitmap: true, + Bitmap: newBitmap, + }) + } else { + delta := store.ComputeDelta(oldBitmap.Bitmap, newBitmap.Bitmap) + err = c.st.InsertNumericBitmapEntry(ctx, store.InsertNumericBitmapEntryParams{ + Name: k.name, + Value: k.value, + Block: c.block, + IsFullBitmap: false, + Bitmap: delta, + }) + } if err != nil { - return fmt.Errorf("failed to upsert numeric attribute %q value %q bitmap: %w", k.name, k.value, err) + return fmt.Errorf("failed to insert numeric bitmap entry %q=%d: %w", k.name, k.value, err) } } + return nil } diff --git a/maintenance.go b/maintenance.go new file mode 100644 index 0000000..e7865c1 --- /dev/null +++ b/maintenance.go @@ -0,0 +1,71 @@ +package sqlitebitmapstore + +import ( + "context" + "database/sql" + "fmt" + + "github.com/Arkiv-Network/sqlite-bitmap-store/store" +) + +// PruneBefore removes all historical data before block B. +// The most recent keyframes at or before B are preserved as the new base. +func (s *SQLiteStore) PruneBefore(ctx context.Context, block uint64) error { + tx, err := s.writePool.BeginTx(ctx, &sql.TxOptions{ + Isolation: sql.LevelSerializable, + ReadOnly: false, + }) + if err != nil { + return fmt.Errorf("failed to begin transaction: %w", err) + } + defer tx.Rollback() + + st := store.New(tx) + blockPtr := &block + + if err := st.PrunePayloadsBefore(ctx, blockPtr); err != nil { + return fmt.Errorf("failed to prune payloads: %w", err) + } + if err := st.PruneStringBitmapsBefore(ctx, block); err != nil { + return fmt.Errorf("failed to prune string bitmaps: %w", err) + } + if err := st.PruneNumericBitmapsBefore(ctx, block); err != nil { + return fmt.Errorf("failed to prune numeric bitmaps: %w", err) + } + + return tx.Commit() +} + +// HandleReorg invalidates all data after block B. Payloads created after B are +// deleted, payloads closed after B are reopened, and bitmap entries after B are +// removed. The last_block is reset to B. +func (s *SQLiteStore) HandleReorg(ctx context.Context, block uint64) error { + tx, err := s.writePool.BeginTx(ctx, &sql.TxOptions{ + Isolation: sql.LevelSerializable, + ReadOnly: false, + }) + if err != nil { + return fmt.Errorf("failed to begin transaction: %w", err) + } + defer tx.Rollback() + + st := store.New(tx) + + if err := st.ReorgReopenPayloads(ctx, block); err != nil { + return fmt.Errorf("failed to reorg payloads: %w", err) + } + if err := st.ReorgDeleteNewPayloads(ctx, block); err != nil { + return fmt.Errorf("failed to delete new payloads: %w", err) + } + if err := st.ReorgDeleteStringBitmaps(ctx, block); err != nil { + return fmt.Errorf("failed to reorg string bitmaps: %w", err) + } + if err := st.ReorgDeleteNumericBitmaps(ctx, block); err != nil { + return fmt.Errorf("failed to reorg numeric bitmaps: %w", err) + } + if err := st.UpsertLastBlock(ctx, block); err != nil { + return fmt.Errorf("failed to update last block: %w", err) + } + + return tx.Commit() +} diff --git a/pusher/push_iterator_test.go b/pusher/push_iterator_test.go index 7db49b6..51d1884 100644 --- a/pusher/push_iterator_test.go +++ b/pusher/push_iterator_test.go @@ -99,7 +99,7 @@ var _ = Describe("PushIterator", func() { var payload []byte var contentType string err = sqlStore.ReadTransaction(ctx, func(q *store.Queries) error { - row, err := q.GetPayloadForEntityKey(ctx, key.Bytes()) + row, err := q.GetCurrentPayloadForEntityKey(ctx, key.Bytes()) if err != nil { return err } @@ -174,13 +174,13 @@ var _ = Describe("PushIterator", func() { Expect(lastBlock).To(Equal(uint64(101))) err = sqlStore.ReadTransaction(ctx, func(q *store.Queries) error { - row1, err := q.GetPayloadForEntityKey(ctx, key1.Bytes()) + row1, err := q.GetCurrentPayloadForEntityKey(ctx, key1.Bytes()) if err != nil { return err } Expect(row1.Payload).To(Equal([]byte("first entity"))) - row2, err := q.GetPayloadForEntityKey(ctx, key2.Bytes()) + row2, err := q.GetCurrentPayloadForEntityKey(ctx, key2.Bytes()) if err != nil { return err } @@ -263,7 +263,7 @@ var _ = Describe("PushIterator", func() { Expect(err).NotTo(HaveOccurred()) err = sqlStore.ReadTransaction(ctx, func(q *store.Queries) error { - row, err := q.GetPayloadForEntityKey(ctx, key.Bytes()) + row, err := q.GetCurrentPayloadForEntityKey(ctx, key.Bytes()) if err != nil { return err } @@ -341,7 +341,7 @@ var _ = Describe("PushIterator", func() { Expect(err).NotTo(HaveOccurred()) err = sqlStore.ReadTransaction(ctx, func(q *store.Queries) error { - _, err := q.GetPayloadForEntityKey(ctx, key.Bytes()) + _, err := q.GetCurrentPayloadForEntityKey(ctx, key.Bytes()) Expect(err).To(HaveOccurred()) return nil }) @@ -388,7 +388,7 @@ var _ = Describe("PushIterator", func() { var originalExpiration uint64 err = sqlStore.ReadTransaction(ctx, func(q *store.Queries) error { - row, err := q.GetPayloadForEntityKey(ctx, key.Bytes()) + row, err := q.GetCurrentPayloadForEntityKey(ctx, key.Bytes()) if err != nil { return err } @@ -428,7 +428,7 @@ var _ = Describe("PushIterator", func() { Expect(err).NotTo(HaveOccurred()) err = sqlStore.ReadTransaction(ctx, func(q *store.Queries) error { - row, err := q.GetPayloadForEntityKey(ctx, key.Bytes()) + row, err := q.GetCurrentPayloadForEntityKey(ctx, key.Bytes()) if err != nil { return err } @@ -479,7 +479,7 @@ var _ = Describe("PushIterator", func() { Expect(err).NotTo(HaveOccurred()) err = sqlStore.ReadTransaction(ctx, func(q *store.Queries) error { - row, err := q.GetPayloadForEntityKey(ctx, key.Bytes()) + row, err := q.GetCurrentPayloadForEntityKey(ctx, key.Bytes()) if err != nil { return err } @@ -518,7 +518,7 @@ var _ = Describe("PushIterator", func() { Expect(err).NotTo(HaveOccurred()) err = sqlStore.ReadTransaction(ctx, func(q *store.Queries) error { - row, err := q.GetPayloadForEntityKey(ctx, key.Bytes()) + row, err := q.GetCurrentPayloadForEntityKey(ctx, key.Bytes()) if err != nil { return err } @@ -597,13 +597,13 @@ var _ = Describe("PushIterator", func() { Expect(lastBlock).To(Equal(uint64(101))) err = sqlStore.ReadTransaction(ctx, func(q *store.Queries) error { - row1, err := q.GetPayloadForEntityKey(ctx, key1.Bytes()) + row1, err := q.GetCurrentPayloadForEntityKey(ctx, key1.Bytes()) if err != nil { return err } Expect(row1.Payload).To(Equal([]byte("batch 1"))) - row2, err := q.GetPayloadForEntityKey(ctx, key2.Bytes()) + row2, err := q.GetCurrentPayloadForEntityKey(ctx, key2.Bytes()) if err != nil { return err } @@ -686,7 +686,7 @@ var _ = Describe("PushIterator", func() { Expect(err).NotTo(HaveOccurred()) err = sqlStore.ReadTransaction(ctx, func(q *store.Queries) error { - row, err := q.GetPayloadForEntityKey(ctx, key.Bytes()) + row, err := q.GetCurrentPayloadForEntityKey(ctx, key.Bytes()) if err != nil { return err } @@ -735,7 +735,7 @@ var _ = Describe("PushIterator", func() { Expect(err).NotTo(HaveOccurred()) err = sqlStore.ReadTransaction(ctx, func(q *store.Queries) error { - row, err := q.GetPayloadForEntityKey(ctx, key.Bytes()) + row, err := q.GetCurrentPayloadForEntityKey(ctx, key.Bytes()) if err != nil { return err } diff --git a/query/evaluate.go b/query/evaluate.go index c9cb3df..5fb26d7 100644 --- a/query/evaluate.go +++ b/query/evaluate.go @@ -2,7 +2,6 @@ package query import ( "context" - "database/sql" "fmt" "github.com/Arkiv-Network/sqlite-bitmap-store/store" @@ -12,9 +11,16 @@ import ( func (t *AST) Evaluate( ctx context.Context, q *store.Queries, + block uint64, ) (*roaring64.Bitmap, error) { if t.Expr == nil { - ids, err := q.EvaluateAll(ctx) + var ids []uint64 + var err error + if block == 0 { + ids, err = q.EvaluateAllCurrent(ctx) + } else { + ids, err = q.EvaluateAllAtBlock(ctx, block) + } if err != nil { return nil, err } @@ -22,24 +28,26 @@ func (t *AST) Evaluate( bm.AddMany(ids) return bm, nil } - return t.Expr.Evaluate(ctx, q) + return t.Expr.Evaluate(ctx, q, block) } func (e *ASTExpr) Evaluate( ctx context.Context, q *store.Queries, + block uint64, ) (*roaring64.Bitmap, error) { - return e.Or.Evaluate(ctx, q) + return e.Or.Evaluate(ctx, q, block) } func (e *ASTOr) Evaluate( ctx context.Context, q *store.Queries, + block uint64, ) (*roaring64.Bitmap, error) { var tmp *roaring64.Bitmap = nil for _, term := range e.Terms { - bm, err := term.Evaluate(ctx, q) + bm, err := term.Evaluate(ctx, q, block) if err != nil { return nil, err } @@ -56,11 +64,12 @@ func (e *ASTOr) Evaluate( func (e *ASTAnd) Evaluate( ctx context.Context, q *store.Queries, + block uint64, ) (*roaring64.Bitmap, error) { var tmp *roaring64.Bitmap = nil for _, term := range e.Terms { - bm, err := term.Evaluate(ctx, q) + bm, err := term.Evaluate(ctx, q, block) if err != nil { return nil, err } @@ -77,337 +86,278 @@ func (e *ASTAnd) Evaluate( func (e *ASTTerm) Evaluate( ctx context.Context, q *store.Queries, + block uint64, ) (*roaring64.Bitmap, error) { switch { case e.Assign != nil: - return e.Assign.Evaluate(ctx, q) + return e.Assign.Evaluate(ctx, q, block) case e.Inclusion != nil: - return e.Inclusion.Evaluate(ctx, q) + return e.Inclusion.Evaluate(ctx, q, block) case e.LessThan != nil: - return e.LessThan.Evaluate(ctx, q) + return e.LessThan.Evaluate(ctx, q, block) case e.LessOrEqualThan != nil: - return e.LessOrEqualThan.Evaluate(ctx, q) + return e.LessOrEqualThan.Evaluate(ctx, q, block) case e.GreaterThan != nil: - return e.GreaterThan.Evaluate(ctx, q) + return e.GreaterThan.Evaluate(ctx, q, block) case e.GreaterOrEqualThan != nil: - return e.GreaterOrEqualThan.Evaluate(ctx, q) + return e.GreaterOrEqualThan.Evaluate(ctx, q, block) case e.Glob != nil: - return e.Glob.Evaluate(ctx, q) + return e.Glob.Evaluate(ctx, q, block) default: return nil, fmt.Errorf("unknown equal expression: %v", e) } } +// reconstructAndOR fetches bitmap chains for each matching value, reconstructs +// each bitmap at the target block, and ORs them together. +func reconstructAndOR(ctx context.Context, q *store.Queries, block uint64, reconstructor func(value string) (*store.Bitmap, error), values []string) (*roaring64.Bitmap, error) { + bm := roaring64.New() + for _, val := range values { + reconstructed, err := reconstructor(val) + if err != nil { + return nil, err + } + if reconstructed != nil && reconstructed.Bitmap != nil { + bm.Or(reconstructed.Bitmap) + } + } + return bm, nil +} + +func reconstructAndORNumeric(ctx context.Context, q *store.Queries, block uint64, name string, values []uint64) (*roaring64.Bitmap, error) { + bm := roaring64.New() + for _, val := range values { + reconstructed, err := q.ReconstructNumericBitmapAtBlock(ctx, name, val, block) + if err != nil { + return nil, err + } + if reconstructed != nil && reconstructed.Bitmap != nil { + bm.Or(reconstructed.Bitmap) + } + } + return bm, nil +} + func (e *Glob) Evaluate( ctx context.Context, q *store.Queries, + block uint64, ) (_ *roaring64.Bitmap, err error) { + eb := store.EffectiveBlock(block) - bm := roaring64.New() - - var bitmaps []*store.Bitmap - + var values []string if e.IsNot { - bitmaps, err = q.EvaluateStringAttributeValueNotGlob(ctx, store.EvaluateStringAttributeValueNotGlobParams{ - Name: e.Var, - Value: e.Value, + values, err = q.GetMatchingStringValuesNotGlob(ctx, store.GetMatchingStringValuesNotGlobParams{ + Name: e.Var, Value: e.Value, TargetBlock: eb, }) - if err != nil { - return nil, err - } } else { - bitmaps, err = q.EvaluateStringAttributeValueGlob(ctx, store.EvaluateStringAttributeValueGlobParams{ - Name: e.Var, - Value: e.Value, + values, err = q.GetMatchingStringValuesGlob(ctx, store.GetMatchingStringValuesGlobParams{ + Name: e.Var, Value: e.Value, TargetBlock: eb, }) - if err != nil { - return nil, err - } } - - for _, bitmap := range bitmaps { - bm.Or(bitmap.Bitmap) + if err != nil { + return nil, err } - return bm, nil + return reconstructAndOR(ctx, q, block, func(val string) (*store.Bitmap, error) { + return q.ReconstructStringBitmapAtBlock(ctx, e.Var, val, block) + }, values) } func (e *LessThan) Evaluate( ctx context.Context, q *store.Queries, + block uint64, ) (_ *roaring64.Bitmap, err error) { - - var bitmaps []*store.Bitmap + eb := store.EffectiveBlock(block) if e.Value.String != nil { - bitmaps, err = q.EvaluateStringAttributeValueLowerThan(ctx, store.EvaluateStringAttributeValueLowerThanParams{ - Name: e.Var, - Value: *e.Value.String, - }) - if err != nil { - return nil, err - } - } else { - bitmaps, err = q.EvaluateNumericAttributeValueLowerThan(ctx, store.EvaluateNumericAttributeValueLowerThanParams{ - Name: e.Var, - Value: *e.Value.Number, + values, err := q.GetMatchingStringValuesLessThan(ctx, store.GetMatchingStringValuesLessThanParams{ + Name: e.Var, Value: *e.Value.String, TargetBlock: eb, }) if err != nil { return nil, err } + return reconstructAndOR(ctx, q, block, func(val string) (*store.Bitmap, error) { + return q.ReconstructStringBitmapAtBlock(ctx, e.Var, val, block) + }, values) } - bm := roaring64.New() - - for _, bitmap := range bitmaps { - bm.Or(bitmap.Bitmap) + values, err := q.GetMatchingNumericValuesLessThan(ctx, store.GetMatchingNumericValuesLessThanParams{ + Name: e.Var, Value: *e.Value.Number, TargetBlock: eb, + }) + if err != nil { + return nil, err } - - return bm, nil + return reconstructAndORNumeric(ctx, q, block, e.Var, values) } func (e *LessOrEqualThan) Evaluate( ctx context.Context, q *store.Queries, + block uint64, ) (_ *roaring64.Bitmap, err error) { - - var bitmaps []*store.Bitmap + eb := store.EffectiveBlock(block) if e.Value.String != nil { - bitmaps, err = q.EvaluateStringAttributeValueLessOrEqualThan(ctx, store.EvaluateStringAttributeValueLessOrEqualThanParams{ - Name: e.Var, - Value: *e.Value.String, - }) - if err != nil { - return nil, err - } - } else { - bitmaps, err = q.EvaluateNumericAttributeValueLessOrEqualThan(ctx, store.EvaluateNumericAttributeValueLessOrEqualThanParams{ - Name: e.Var, - Value: *e.Value.Number, + values, err := q.GetMatchingStringValuesLessOrEqualThan(ctx, store.GetMatchingStringValuesLessOrEqualThanParams{ + Name: e.Var, Value: *e.Value.String, TargetBlock: eb, }) if err != nil { return nil, err } + return reconstructAndOR(ctx, q, block, func(val string) (*store.Bitmap, error) { + return q.ReconstructStringBitmapAtBlock(ctx, e.Var, val, block) + }, values) } - bm := roaring64.New() - - for _, bitmap := range bitmaps { - bm.Or(bitmap.Bitmap) + values, err := q.GetMatchingNumericValuesLessOrEqualThan(ctx, store.GetMatchingNumericValuesLessOrEqualThanParams{ + Name: e.Var, Value: *e.Value.Number, TargetBlock: eb, + }) + if err != nil { + return nil, err } - - return bm, nil + return reconstructAndORNumeric(ctx, q, block, e.Var, values) } func (e *GreaterThan) Evaluate( ctx context.Context, q *store.Queries, + block uint64, ) (_ *roaring64.Bitmap, err error) { - - var bitmaps []*store.Bitmap + eb := store.EffectiveBlock(block) if e.Value.String != nil { - bitmaps, err = q.EvaluateStringAttributeValueGreaterThan(ctx, store.EvaluateStringAttributeValueGreaterThanParams{ - Name: e.Var, - Value: *e.Value.String, - }) - if err != nil { - return nil, err - } - - } else { - bitmaps, err = q.EvaluateNumericAttributeValueGreaterThan(ctx, store.EvaluateNumericAttributeValueGreaterThanParams{ - Name: e.Var, - Value: *e.Value.Number, + values, err := q.GetMatchingStringValuesGreaterThan(ctx, store.GetMatchingStringValuesGreaterThanParams{ + Name: e.Var, Value: *e.Value.String, TargetBlock: eb, }) if err != nil { return nil, err } + return reconstructAndOR(ctx, q, block, func(val string) (*store.Bitmap, error) { + return q.ReconstructStringBitmapAtBlock(ctx, e.Var, val, block) + }, values) } - bm := roaring64.New() - - for _, bitmap := range bitmaps { - bm.Or(bitmap.Bitmap) + values, err := q.GetMatchingNumericValuesGreaterThan(ctx, store.GetMatchingNumericValuesGreaterThanParams{ + Name: e.Var, Value: *e.Value.Number, TargetBlock: eb, + }) + if err != nil { + return nil, err } - - return bm, nil + return reconstructAndORNumeric(ctx, q, block, e.Var, values) } func (e *GreaterOrEqualThan) Evaluate( ctx context.Context, q *store.Queries, + block uint64, ) (_ *roaring64.Bitmap, err error) { - - var bitmaps []*store.Bitmap + eb := store.EffectiveBlock(block) if e.Value.String != nil { - bitmaps, err = q.EvaluateStringAttributeValueGreaterOrEqualThan(ctx, store.EvaluateStringAttributeValueGreaterOrEqualThanParams{ - Name: e.Var, - Value: *e.Value.String, - }) - if err != nil { - return nil, err - } - - } else { - bitmaps, err = q.EvaluateNumericAttributeValueGreaterOrEqualThan(ctx, store.EvaluateNumericAttributeValueGreaterOrEqualThanParams{ - Name: e.Var, - Value: *e.Value.Number, + values, err := q.GetMatchingStringValuesGreaterOrEqualThan(ctx, store.GetMatchingStringValuesGreaterOrEqualThanParams{ + Name: e.Var, Value: *e.Value.String, TargetBlock: eb, }) if err != nil { return nil, err } + return reconstructAndOR(ctx, q, block, func(val string) (*store.Bitmap, error) { + return q.ReconstructStringBitmapAtBlock(ctx, e.Var, val, block) + }, values) } - bm := roaring64.New() - - for _, bitmap := range bitmaps { - bm.Or(bitmap.Bitmap) + values, err := q.GetMatchingNumericValuesGreaterOrEqualThan(ctx, store.GetMatchingNumericValuesGreaterOrEqualThanParams{ + Name: e.Var, Value: *e.Value.Number, TargetBlock: eb, + }) + if err != nil { + return nil, err } - - return bm, nil + return reconstructAndORNumeric(ctx, q, block, e.Var, values) } func (e *Equality) Evaluate( ctx context.Context, q *store.Queries, + block uint64, ) (_ *roaring64.Bitmap, err error) { + eb := store.EffectiveBlock(block) if e.Value.String != nil { - if e.IsNot { - - var bitmaps []*store.Bitmap - bitmaps, err = q.EvaluateStringAttributeValueNotEqual(ctx, store.EvaluateStringAttributeValueNotEqualParams{ - Name: e.Var, - Value: *e.Value.String, + values, err := q.GetMatchingStringValuesNotEqual(ctx, store.GetMatchingStringValuesNotEqualParams{ + Name: e.Var, Value: *e.Value.String, TargetBlock: eb, }) if err != nil { return nil, err } - - bm := roaring64.New() - - for _, bitmap := range bitmaps { - bm.Or(bitmap.Bitmap) - } - - return bm, nil - - } else { - bm, err := q.EvaluateStringAttributeValueEqual(ctx, store.EvaluateStringAttributeValueEqualParams{ - Name: e.Var, - Value: *e.Value.String, - }) - - if err == sql.ErrNoRows { - return roaring64.New(), nil - } - - if err != nil { - return nil, err - } - - return bm.Bitmap, nil + return reconstructAndOR(ctx, q, block, func(val string) (*store.Bitmap, error) { + return q.ReconstructStringBitmapAtBlock(ctx, e.Var, val, block) + }, values) } - } else { - if e.IsNot { - var bitmaps []*store.Bitmap - bitmaps, err = q.EvaluateNumericAttributeValueNotEqual(ctx, store.EvaluateNumericAttributeValueNotEqualParams{ - Name: e.Var, - Value: *e.Value.Number, - }) - if err != nil { - return nil, err - } - - bm := roaring64.New() - for _, bitmap := range bitmaps { - bm.Or(bitmap.Bitmap) - } - - return bm, nil - } else { - bitmap, err := q.EvaluateNumericAttributeValueEqual(ctx, store.EvaluateNumericAttributeValueEqualParams{ - Name: e.Var, - Value: *e.Value.Number, - }) - - if err == sql.ErrNoRows { - return roaring64.New(), nil - } - - if err != nil { - return nil, err - } + bm, err := q.ReconstructStringBitmapAtBlock(ctx, e.Var, *e.Value.String, block) + if err != nil { + return nil, err + } + return bm.Bitmap, nil + } - return bitmap.Bitmap, nil + if e.IsNot { + values, err := q.GetMatchingNumericValuesNotEqual(ctx, store.GetMatchingNumericValuesNotEqualParams{ + Name: e.Var, Value: *e.Value.Number, TargetBlock: eb, + }) + if err != nil { + return nil, err } + return reconstructAndORNumeric(ctx, q, block, e.Var, values) } + bm, err := q.ReconstructNumericBitmapAtBlock(ctx, e.Var, *e.Value.Number, block) + if err != nil { + return nil, err + } + return bm.Bitmap, nil } func (e *Inclusion) Evaluate( ctx context.Context, q *store.Queries, + block uint64, ) (_ *roaring64.Bitmap, err error) { + eb := store.EffectiveBlock(block) if len(e.Values.Strings) != 0 { - - var bitmaps []*store.Bitmap - + var values []string if e.IsNot { - bitmaps, err = q.EvaluateStringAttributeValueNotInclusion(ctx, store.EvaluateStringAttributeValueNotInclusionParams{ - Name: e.Var, - Values: e.Values.Strings, + values, err = q.GetMatchingStringValuesNotInclusion(ctx, store.GetMatchingStringValuesNotInclusionParams{ + Name: e.Var, Values: e.Values.Strings, TargetBlock: eb, }) - if err != nil { - return nil, err - } } else { - - bitmaps, err = q.EvaluateStringAttributeValueInclusion(ctx, store.EvaluateStringAttributeValueInclusionParams{ - Name: e.Var, - Values: e.Values.Strings, + values, err = q.GetMatchingStringValuesInclusion(ctx, store.GetMatchingStringValuesInclusionParams{ + Name: e.Var, Values: e.Values.Strings, TargetBlock: eb, }) - if err != nil { - return nil, err - } } - bm := roaring64.New() - for _, bitmap := range bitmaps { - bm.Or(bitmap.Bitmap) + if err != nil { + return nil, err } - return bm, nil + return reconstructAndOR(ctx, q, block, func(val string) (*store.Bitmap, error) { + return q.ReconstructStringBitmapAtBlock(ctx, e.Var, val, block) + }, values) + } + var values []uint64 + if e.IsNot { + values, err = q.GetMatchingNumericValuesNotInclusion(ctx, store.GetMatchingNumericValuesNotInclusionParams{ + Name: e.Var, Values: e.Values.Numbers, TargetBlock: eb, + }) } else { - var bitmaps []*store.Bitmap - - if e.IsNot { - bitmaps, err = q.EvaluateNumericAttributeValueNotInclusion(ctx, store.EvaluateNumericAttributeValueNotInclusionParams{ - Name: e.Var, - Values: e.Values.Numbers, - }) - if err != nil { - return nil, err - } - } else { - bitmaps, err = q.EvaluateNumericAttributeValueInclusion(ctx, store.EvaluateNumericAttributeValueInclusionParams{ - Name: e.Var, - Values: e.Values.Numbers, - }) - if err != nil { - return nil, err - } - } - bm := roaring64.New() - for _, bitmap := range bitmaps { - bm.Or(bitmap.Bitmap) - } - return bm, nil + values, err = q.GetMatchingNumericValuesInclusion(ctx, store.GetMatchingNumericValuesInclusionParams{ + Name: e.Var, Values: e.Values.Numbers, TargetBlock: eb, + }) } - + if err != nil { + return nil, err + } + return reconstructAndORNumeric(ctx, q, block, e.Var, values) } diff --git a/query_rpc.go b/query_rpc.go index d8d3ea9..9e64bfe 100644 --- a/query_rpc.go +++ b/query_rpc.go @@ -157,6 +157,7 @@ func (s *SQLiteStore) QueryEntities( bitmap, err := q.Evaluate( ctx, queries, + options.GetAtBlock(), ) if err != nil { return fmt.Errorf("error evaluating query: %w", err) diff --git a/sqlitestore.go b/sqlitestore.go index ca631b7..dee8771 100644 --- a/sqlitestore.go +++ b/sqlitestore.go @@ -25,7 +25,6 @@ import ( ) var ( - // Metrics for tracking operations metricOperationStarted = metrics.NewRegisteredCounter("arkiv_store/operations_started", nil) metricOperationSuccessful = metrics.NewRegisteredCounter("arkiv_store/operations_successful", nil) metricCreates = metrics.NewRegisteredCounter("arkiv_store/creates", nil) @@ -36,10 +35,7 @@ var ( metricDeletesBytes = metrics.NewRegisteredCounter("arkiv_store/deletes_bytes", nil) metricExtends = metrics.NewRegisteredCounter("arkiv_store/extends", nil) metricOwnerChanges = metrics.NewRegisteredCounter("arkiv_store/owner_changes", nil) - // Tracks operation duration (ms) using an exponential decay sample so the histogram - // is more responsive to recent performance by weighting newer measurements higher - // (sample size 100, alpha 0.4). - metricOperationTime = metrics.NewRegisteredHistogram("arkiv_store/operation_time_ms", nil, metrics.NewExpDecaySample(100, 0.4)) + metricOperationTime = metrics.NewRegisteredHistogram("arkiv_store/operation_time_ms", nil, metrics.NewExpDecaySample(100, 0.4)) ) type SQLiteStore struct { @@ -136,7 +132,6 @@ func (s *SQLiteStore) FollowEvents(ctx context.Context, iterator arkivevents.Bat return fmt.Errorf("failed to follow events: %w", batch.Error) } - // We will calculate totals for the log at the end, but track per-block for metrics stats := make(map[uint64]*blockStats) err := func() error { @@ -161,8 +156,6 @@ func (s *SQLiteStore) FollowEvents(ctx context.Context, iterator arkivevents.Bat return fmt.Errorf("failed to get last block from database: %w", err) } - cache := newBitmapCache(st) - startTime := time.Now() metricOperationStarted.Inc(1) @@ -175,12 +168,14 @@ func (s *SQLiteStore) FollowEvents(ctx context.Context, iterator arkivevents.Bat continue mainLoop } - // Initialize stats for this block if _, ok := stats[block.Number]; !ok { stats[block.Number] = &blockStats{} } blockStat := stats[block.Number] + // Per-block bitmap cache for bitemporality. + cache := newBitmapCache(st, block.Number) + updatesMap := map[common.Hash][]*events.OPUpdate{} for _, operation := range block.Operations { @@ -197,13 +192,11 @@ func (s *SQLiteStore) FollowEvents(ctx context.Context, iterator arkivevents.Bat switch { case operation.Create != nil: - // expiresAtBlock := blockNumber + operation.Create.BTL blockStat.creates++ blockStat.createsBytes += int64(len(operation.Create.Content)) key := operation.Create.Key stringAttributes := maps.Clone(operation.Create.StringAttributes) - stringAttributes["$owner"] = strings.ToLower(operation.Create.Owner.Hex()) stringAttributes["$creator"] = strings.ToLower(operation.Create.Owner.Hex()) stringAttributes["$key"] = strings.ToLower(key.Hex()) @@ -219,14 +212,21 @@ func (s *SQLiteStore) FollowEvents(ctx context.Context, iterator arkivevents.Bat numericAttributes["$txIndex"] = uint64(operation.TxIndex) numericAttributes["$opIndex"] = uint64(operation.OpIndex) - id, err := st.UpsertPayload( + // Close any existing version (handles re-creation after delete). + _ = st.ClosePayloadVersion(ctx, store.ClosePayloadVersionParams{ + Block: &block.Number, + EntityKey: operation.Create.Key.Bytes(), + }) + + id, err := st.InsertPayload( ctx, - store.UpsertPayloadParams{ + store.InsertPayloadParams{ EntityKey: operation.Create.Key.Bytes(), Payload: operation.Create.Content, ContentType: operation.Create.ContentType, StringAttributes: store.NewStringAttributes(stringAttributes), NumericAttributes: store.NewNumericAttributes(numericAttributes), + FromBlock: block.Number, }, ) if err != nil { @@ -241,20 +241,17 @@ func (s *SQLiteStore) FollowEvents(ctx context.Context, iterator arkivevents.Bat } for k, v := range numericAttributes { - - // skip txIndex and opIndex because they are not used for querying switch k { case "$txIndex", "$opIndex": continue } - err = cache.AddToNumericBitmap(ctx, k, v, id) if err != nil { return fmt.Errorf("failed to add numeric attribute value bitmap: %w", err) } } - case operation.Update != nil: + case operation.Update != nil: updates := updatesMap[operation.Update.Key] lastUpdate := updates[len(updates)-1] @@ -266,17 +263,16 @@ func (s *SQLiteStore) FollowEvents(ctx context.Context, iterator arkivevents.Bat key := operation.Update.Key.Bytes() - latestPayload, err := st.GetPayloadForEntityKey(ctx, key) + latestPayload, err := st.GetCurrentPayloadForEntityKey(ctx, key) if err != nil { return fmt.Errorf("failed to get latest payload: %w", err) } + oldID := latestPayload.ID oldStringAttributes := latestPayload.StringAttributes - oldNumericAttributes := latestPayload.NumericAttributes stringAttributes := maps.Clone(operation.Update.StringAttributes) - stringAttributes["$owner"] = strings.ToLower(operation.Update.Owner.Hex()) stringAttributes["$creator"] = oldStringAttributes.Values["$creator"] stringAttributes["$key"] = strings.ToLower(operation.Update.Key.Hex()) @@ -285,20 +281,28 @@ func (s *SQLiteStore) FollowEvents(ctx context.Context, iterator arkivevents.Bat numericAttributes := maps.Clone(operation.Update.NumericAttributes) numericAttributes["$expiration"] = uint64(untilBlock) numericAttributes["$createdAtBlock"] = oldNumericAttributes.Values["$createdAtBlock"] - numericAttributes["$sequence"] = oldNumericAttributes.Values["$sequence"] numericAttributes["$txIndex"] = oldNumericAttributes.Values["$txIndex"] numericAttributes["$opIndex"] = oldNumericAttributes.Values["$opIndex"] numericAttributes["$lastModifiedAtBlock"] = uint64(block.Number) - id, err := st.UpsertPayload( + err = st.ClosePayloadVersion(ctx, store.ClosePayloadVersionParams{ + Block: &block.Number, + EntityKey: key, + }) + if err != nil { + return fmt.Errorf("failed to close payload version: %w", err) + } + + newID, err := st.InsertPayload( ctx, - store.UpsertPayloadParams{ + store.InsertPayloadParams{ EntityKey: key, Payload: operation.Update.Content, ContentType: operation.Update.ContentType, StringAttributes: store.NewStringAttributes(stringAttributes), NumericAttributes: store.NewNumericAttributes(numericAttributes), + FromBlock: block.Number, }, ) if err != nil { @@ -306,49 +310,42 @@ func (s *SQLiteStore) FollowEvents(ctx context.Context, iterator arkivevents.Bat } for k, v := range oldStringAttributes.Values { - err = cache.RemoveFromStringBitmap(ctx, k, v, id) + err = cache.RemoveFromStringBitmap(ctx, k, v, oldID) if err != nil { return fmt.Errorf("failed to remove string attribute value bitmap: %w", err) } } for k, v := range oldNumericAttributes.Values { - // skip txIndex and opIndex because they are not used for querying switch k { case "$txIndex", "$opIndex": continue } - - err = cache.RemoveFromNumericBitmap(ctx, k, v, id) + err = cache.RemoveFromNumericBitmap(ctx, k, v, oldID) if err != nil { return fmt.Errorf("failed to remove numeric attribute value bitmap: %w", err) } } - // TODO: delete entity from the indexes - for k, v := range stringAttributes { - err = cache.AddToStringBitmap(ctx, k, v, id) + err = cache.AddToStringBitmap(ctx, k, v, newID) if err != nil { return fmt.Errorf("failed to add string attribute value bitmap: %w", err) } } for k, v := range numericAttributes { - // skip txIndex and opIndex because they are not used for querying switch k { case "$txIndex", "$opIndex": continue } - - err = cache.AddToNumericBitmap(ctx, k, v, id) + err = cache.AddToNumericBitmap(ctx, k, v, newID) if err != nil { return fmt.Errorf("failed to add numeric attribute value bitmap: %w", err) } } case operation.Delete != nil || operation.Expire != nil: - blockStat.deletes++ var key []byte if operation.Delete != nil { @@ -357,78 +354,80 @@ func (s *SQLiteStore) FollowEvents(ctx context.Context, iterator arkivevents.Bat key = common.Hash(*operation.Expire).Bytes() } - latestPayload, err := st.GetPayloadForEntityKey(ctx, key) + latestPayload, err := st.GetCurrentPayloadForEntityKey(ctx, key) if err != nil { return fmt.Errorf("failed to get latest payload: %w", err) } blockStat.deletesBytes += int64(len(latestPayload.Payload)) - oldStringAttributes := latestPayload.StringAttributes - - oldNumericAttributes := latestPayload.NumericAttributes - - for k, v := range oldStringAttributes.Values { + for k, v := range latestPayload.StringAttributes.Values { err = cache.RemoveFromStringBitmap(ctx, k, v, latestPayload.ID) if err != nil { return fmt.Errorf("failed to remove string attribute value bitmap: %w", err) } } - for k, v := range oldNumericAttributes.Values { - // skip txIndex and opIndex because they are not used for querying + for k, v := range latestPayload.NumericAttributes.Values { switch k { case "$txIndex", "$opIndex": continue } - err = cache.RemoveFromNumericBitmap(ctx, k, v, latestPayload.ID) if err != nil { return fmt.Errorf("failed to remove numeric attribute value bitmap: %w", err) } } - err = st.DeletePayloadForEntityKey(ctx, key) + err = st.ClosePayloadVersion(ctx, store.ClosePayloadVersionParams{ + Block: &block.Number, + EntityKey: key, + }) if err != nil { - return fmt.Errorf("failed to delete payload: %w", err) + return fmt.Errorf("failed to close payload version: %w", err) } case operation.ExtendBTL != nil: - blockStat.extends++ key := operation.ExtendBTL.Key.Bytes() - latestPayload, err := st.GetPayloadForEntityKey(ctx, key) + latestPayload, err := st.GetCurrentPayloadForEntityKey(ctx, key) if err != nil { return fmt.Errorf("failed to get latest payload: %w", err) } - - oldNumericAttributes := latestPayload.NumericAttributes - - oldExpiration := oldNumericAttributes.Values["$expiration"] - + oldID := latestPayload.ID + oldExpiration := latestPayload.NumericAttributes.Values["$expiration"] newToBlock := oldExpiration + operation.ExtendBTL.BTL - numericAttributes := maps.Clone(oldNumericAttributes.Values) + numericAttributes := maps.Clone(latestPayload.NumericAttributes.Values) numericAttributes["$expiration"] = uint64(newToBlock) - id, err := st.UpsertPayload(ctx, store.UpsertPayloadParams{ + err = st.ClosePayloadVersion(ctx, store.ClosePayloadVersionParams{ + Block: &block.Number, + EntityKey: key, + }) + if err != nil { + return fmt.Errorf("failed to close payload version: %w", err) + } + + newID, err := st.InsertPayload(ctx, store.InsertPayloadParams{ EntityKey: key, Payload: latestPayload.Payload, ContentType: latestPayload.ContentType, StringAttributes: latestPayload.StringAttributes, NumericAttributes: store.NewNumericAttributes(numericAttributes), + FromBlock: block.Number, }) if err != nil { return fmt.Errorf("failed to insert payload at block %d txIndex %d opIndex %d: %w", block.Number, operation.TxIndex, operation.OpIndex, err) } - err = cache.RemoveFromNumericBitmap(ctx, "$expiration", oldExpiration, id) + err = cache.RemoveFromNumericBitmap(ctx, "$expiration", oldExpiration, oldID) if err != nil { return fmt.Errorf("failed to remove numeric attribute value bitmap: %w", err) } - err = cache.AddToNumericBitmap(ctx, "$expiration", newToBlock, id) + err = cache.AddToNumericBitmap(ctx, "$expiration", newToBlock, newID) if err != nil { return fmt.Errorf("failed to add numeric attribute value bitmap: %w", err) } @@ -437,39 +436,46 @@ func (s *SQLiteStore) FollowEvents(ctx context.Context, iterator arkivevents.Bat blockStat.ownerChanges++ key := operation.ChangeOwner.Key.Bytes() - latestPayload, err := st.GetPayloadForEntityKey(ctx, key) + latestPayload, err := st.GetCurrentPayloadForEntityKey(ctx, key) if err != nil { return fmt.Errorf("failed to get latest payload: %w", err) } - - stringAttributes := latestPayload.StringAttributes - - oldOwner := stringAttributes.Values["$owner"] - + oldID := latestPayload.ID + oldOwner := latestPayload.StringAttributes.Values["$owner"] newOwner := strings.ToLower(operation.ChangeOwner.Owner.Hex()) - stringAttributes.Values["$owner"] = newOwner + newStringAttrs := &store.StringAttributes{Values: maps.Clone(latestPayload.StringAttributes.Values)} + newStringAttrs.Values["$owner"] = newOwner + + err = st.ClosePayloadVersion(ctx, store.ClosePayloadVersionParams{ + Block: &block.Number, + EntityKey: key, + }) + if err != nil { + return fmt.Errorf("failed to close payload version: %w", err) + } - id, err := st.UpsertPayload( + newID, err := st.InsertPayload( ctx, - store.UpsertPayloadParams{ + store.InsertPayloadParams{ EntityKey: key, Payload: latestPayload.Payload, ContentType: latestPayload.ContentType, - StringAttributes: stringAttributes, + StringAttributes: newStringAttrs, NumericAttributes: latestPayload.NumericAttributes, + FromBlock: block.Number, }, ) if err != nil { return fmt.Errorf("failed to insert payload at block %d txIndex %d opIndex %d: %w", block.Number, operation.TxIndex, operation.OpIndex, err) } - err = cache.RemoveFromStringBitmap(ctx, "$owner", oldOwner, id) + err = cache.RemoveFromStringBitmap(ctx, "$owner", oldOwner, oldID) if err != nil { return fmt.Errorf("failed to remove string attribute value bitmap for owner: %w", err) } - err = cache.AddToStringBitmap(ctx, "$owner", newOwner, id) + err = cache.AddToStringBitmap(ctx, "$owner", newOwner, newID) if err != nil { return fmt.Errorf("failed to add string attribute value bitmap for owner: %w", err) } @@ -480,7 +486,11 @@ func (s *SQLiteStore) FollowEvents(ctx context.Context, iterator arkivevents.Bat } - // Log per block if needed, but we can now rely on the map for totals later + err = cache.Flush(ctx) + if err != nil { + return fmt.Errorf("failed to flush bitmap cache for block %d: %w", block.Number, err) + } + s.log.Info("block updated", "block", block.Number, "creates", blockStat.creates, "updates", blockStat.updates, "deletes", blockStat.deletes, "extends", blockStat.extends, "ownerChanges", blockStat.ownerChanges) } @@ -489,17 +499,11 @@ func (s *SQLiteStore) FollowEvents(ctx context.Context, iterator arkivevents.Bat return fmt.Errorf("failed to upsert last block: %w", err) } - err = cache.Flush(ctx) - if err != nil { - return fmt.Errorf("failed to flush bitmap cache: %w", err) - } - err = tx.Commit() if err != nil { return fmt.Errorf("failed to commit transaction: %w", err) } - // Calculate batch totals for logging and update metrics PER BLOCK var ( totalCreates int64 totalCreatesBytes int64 @@ -511,7 +515,6 @@ func (s *SQLiteStore) FollowEvents(ctx context.Context, iterator arkivevents.Bat totalOwnerChanges int64 ) - // Iterate blocks again to preserve order and update metrics per block for _, block := range batch.Batch.Blocks { if stat, ok := stats[block.Number]; ok { totalCreates += stat.creates @@ -523,7 +526,6 @@ func (s *SQLiteStore) FollowEvents(ctx context.Context, iterator arkivevents.Bat totalExtends += stat.extends totalOwnerChanges += stat.ownerChanges - // Update metrics specifically per block if stat.creates > 0 { metricCreates.Inc(stat.creates) } diff --git a/sqlitestore_test.go b/sqlitestore_test.go index bf0c8f0..91e4f3d 100644 --- a/sqlitestore_test.go +++ b/sqlitestore_test.go @@ -126,10 +126,7 @@ var _ = Describe("SQLiteStore", func() { err = sqlStore.ReadTransaction(ctx, func(q *store.Queries) error { // Query by string attribute: type = "document" - docBitmap, err := q.EvaluateStringAttributeValueEqual(ctx, store.EvaluateStringAttributeValueEqualParams{ - Name: "type", - Value: "document", - }) + docBitmap, err := q.ReconstructLatestStringBitmap(ctx, "type", "document") Expect(err).NotTo(HaveOccurred()) Expect(docBitmap).NotTo(BeNil()) @@ -144,10 +141,7 @@ var _ = Describe("SQLiteStore", func() { Expect(docPayloads[0].StringAttributes.Values["type"]).To(Equal("document")) // Query by string attribute: type = "image" - imageBitmap, err := q.EvaluateStringAttributeValueEqual(ctx, store.EvaluateStringAttributeValueEqualParams{ - Name: "type", - Value: "image", - }) + imageBitmap, err := q.ReconstructLatestStringBitmap(ctx, "type", "image") Expect(err).NotTo(HaveOccurred()) Expect(imageBitmap).NotTo(BeNil()) @@ -161,10 +155,7 @@ var _ = Describe("SQLiteStore", func() { Expect(imagePayloads[0].ContentType).To(Equal("image/png")) // Query by numeric attribute: version = 1 - version1Bitmap, err := q.EvaluateNumericAttributeValueEqual(ctx, store.EvaluateNumericAttributeValueEqualParams{ - Name: "version", - Value: 1, - }) + version1Bitmap, err := q.ReconstructLatestNumericBitmap(ctx, "version", 1) Expect(err).NotTo(HaveOccurred()) Expect(version1Bitmap).NotTo(BeNil()) @@ -176,40 +167,32 @@ var _ = Describe("SQLiteStore", func() { Expect(version1Payloads).To(HaveLen(1)) Expect(version1Payloads[0].NumericAttributes.Values["version"]).To(Equal(uint64(1))) - // Query by numeric attribute: version > 1 - versionGT1Bitmaps, err := q.EvaluateNumericAttributeValueGreaterThan(ctx, store.EvaluateNumericAttributeValueGreaterThanParams{ - Name: "version", - Value: 1, - }) + // Query by numeric attribute: version = 2 (replaces version > 1 test) + version2Bitmap, err := q.ReconstructLatestNumericBitmap(ctx, "version", 2) Expect(err).NotTo(HaveOccurred()) - Expect(versionGT1Bitmaps).To(HaveLen(1)) + Expect(version2Bitmap).NotTo(BeNil()) - // Combine bitmaps to get all IDs with version > 1 - combinedBitmap := store.NewBitmap() - for _, bm := range versionGT1Bitmaps { - combinedBitmap.Or(bm.Bitmap) - } + version2IDs := version2Bitmap.ToArray() + Expect(version2IDs).To(HaveLen(1)) - versionGT1IDs := combinedBitmap.ToArray() - Expect(versionGT1IDs).To(HaveLen(1)) + version2Payloads, err := q.RetrievePayloads(ctx, version2IDs) + Expect(err).NotTo(HaveOccurred()) + Expect(version2Payloads).To(HaveLen(1)) + Expect(version2Payloads[0].NumericAttributes.Values["version"]).To(Equal(uint64(2))) - versionGT1Payloads, err := q.RetrievePayloads(ctx, versionGT1IDs) + // Query by numeric attribute: priority = 10 and priority = 20 (replaces priority >= 10 test) + priority10Bitmap, err := q.ReconstructLatestNumericBitmap(ctx, "priority", 10) Expect(err).NotTo(HaveOccurred()) - Expect(versionGT1Payloads).To(HaveLen(1)) - Expect(versionGT1Payloads[0].NumericAttributes.Values["version"]).To(Equal(uint64(2))) + Expect(priority10Bitmap.GetCardinality()).To(Equal(uint64(1))) - // Query by numeric attribute: priority >= 10 - priorityGTE10Bitmaps, err := q.EvaluateNumericAttributeValueGreaterOrEqualThan(ctx, store.EvaluateNumericAttributeValueGreaterOrEqualThanParams{ - Name: "priority", - Value: 10, - }) + priority20Bitmap, err := q.ReconstructLatestNumericBitmap(ctx, "priority", 20) Expect(err).NotTo(HaveOccurred()) - Expect(priorityGTE10Bitmaps).To(HaveLen(2)) + Expect(priority20Bitmap.GetCardinality()).To(Equal(uint64(1))) + // Combine both priority bitmaps priorityCombined := store.NewBitmap() - for _, bm := range priorityGTE10Bitmaps { - priorityCombined.Or(bm.Bitmap) - } + priorityCombined.Or(priority10Bitmap.Bitmap) + priorityCombined.Or(priority20Bitmap.Bitmap) priorityIDs := priorityCombined.ToArray() Expect(priorityIDs).To(HaveLen(2)) @@ -307,7 +290,7 @@ var _ = Describe("SQLiteStore", func() { // Verify the update err = sqlStore.ReadTransaction(ctx, func(q *store.Queries) error { - row, err := q.GetPayloadForEntityKey(ctx, key.Bytes()) + row, err := q.GetCurrentPayloadForEntityKey(ctx, key.Bytes()) Expect(err).NotTo(HaveOccurred()) Expect(row.Payload).To(Equal([]byte(`{"updated": true}`))) @@ -319,21 +302,15 @@ var _ = Describe("SQLiteStore", func() { Expect(row.NumericAttributes.Values["$createdAtBlock"]).To(Equal(uint64(100))) // Verify old bitmap index is removed - oldStatusBitmap, err := q.EvaluateStringAttributeValueEqual(ctx, store.EvaluateStringAttributeValueEqualParams{ - Name: "status", - Value: "draft", - }) - Expect(err).To(HaveOccurred()) // Should not find old value + oldStatusBitmap, err := q.ReconstructLatestStringBitmap(ctx, "status", "draft") + Expect(err).NotTo(HaveOccurred()) + Expect(oldStatusBitmap.IsEmpty()).To(BeTrue()) // Verify new bitmap index exists - newStatusBitmap, err := q.EvaluateStringAttributeValueEqual(ctx, store.EvaluateStringAttributeValueEqualParams{ - Name: "status", - Value: "published", - }) + newStatusBitmap, err := q.ReconstructLatestStringBitmap(ctx, "status", "published") Expect(err).NotTo(HaveOccurred()) Expect(newStatusBitmap.ToArray()).To(HaveLen(1)) - _ = oldStatusBitmap return nil }) Expect(err).NotTo(HaveOccurred()) @@ -385,7 +362,7 @@ var _ = Describe("SQLiteStore", func() { // Verify entity exists err = sqlStore.ReadTransaction(ctx, func(q *store.Queries) error { - _, err := q.GetPayloadForEntityKey(ctx, key.Bytes()) + _, err := q.GetCurrentPayloadForEntityKey(ctx, key.Bytes()) Expect(err).NotTo(HaveOccurred()) return nil }) @@ -420,15 +397,13 @@ var _ = Describe("SQLiteStore", func() { // Verify entity is deleted err = sqlStore.ReadTransaction(ctx, func(q *store.Queries) error { - _, err := q.GetPayloadForEntityKey(ctx, key.Bytes()) + _, err := q.GetCurrentPayloadForEntityKey(ctx, key.Bytes()) Expect(err).To(HaveOccurred()) // Verify bitmap index is removed - _, err = q.EvaluateStringAttributeValueEqual(ctx, store.EvaluateStringAttributeValueEqualParams{ - Name: "deletable", - Value: "yes", - }) - Expect(err).To(HaveOccurred()) + deletableBitmap, err := q.ReconstructLatestStringBitmap(ctx, "deletable", "yes") + Expect(err).NotTo(HaveOccurred()) + Expect(deletableBitmap.IsEmpty()).To(BeTrue()) return nil }) @@ -506,7 +481,7 @@ var _ = Describe("SQLiteStore", func() { // Verify entity is expired (deleted) err = sqlStore.ReadTransaction(ctx, func(q *store.Queries) error { - _, err := q.GetPayloadForEntityKey(ctx, key.Bytes()) + _, err := q.GetCurrentPayloadForEntityKey(ctx, key.Bytes()) Expect(err).To(HaveOccurred()) return nil }) @@ -556,7 +531,7 @@ var _ = Describe("SQLiteStore", func() { // Verify original expiration var originalExpiration uint64 err = sqlStore.ReadTransaction(ctx, func(q *store.Queries) error { - row, err := q.GetPayloadForEntityKey(ctx, key.Bytes()) + row, err := q.GetCurrentPayloadForEntityKey(ctx, key.Bytes()) Expect(err).NotTo(HaveOccurred()) originalExpiration = row.NumericAttributes.Values["$expiration"] Expect(originalExpiration).To(Equal(uint64(600))) // 100 + 500 @@ -595,27 +570,21 @@ var _ = Describe("SQLiteStore", func() { // Verify extended expiration err = sqlStore.ReadTransaction(ctx, func(q *store.Queries) error { - row, err := q.GetPayloadForEntityKey(ctx, key.Bytes()) + row, err := q.GetCurrentPayloadForEntityKey(ctx, key.Bytes()) Expect(err).NotTo(HaveOccurred()) newExpiration := row.NumericAttributes.Values["$expiration"] Expect(newExpiration).To(Equal(uint64(1600))) // Verify old expiration bitmap is removed - oldExpBitmap, err := q.EvaluateNumericAttributeValueEqual(ctx, store.EvaluateNumericAttributeValueEqualParams{ - Name: "$expiration", - Value: 600, - }) - Expect(err).To(HaveOccurred()) + oldExpBitmap, err := q.ReconstructLatestNumericBitmap(ctx, "$expiration", 600) + Expect(err).NotTo(HaveOccurred()) + Expect(oldExpBitmap.IsEmpty()).To(BeTrue()) // Verify new expiration bitmap exists - newExpBitmap, err := q.EvaluateNumericAttributeValueEqual(ctx, store.EvaluateNumericAttributeValueEqualParams{ - Name: "$expiration", - Value: 1600, - }) + newExpBitmap, err := q.ReconstructLatestNumericBitmap(ctx, "$expiration", 1600) Expect(err).NotTo(HaveOccurred()) Expect(newExpBitmap.ToArray()).To(HaveLen(1)) - _ = oldExpBitmap return nil }) Expect(err).NotTo(HaveOccurred()) @@ -664,7 +633,7 @@ var _ = Describe("SQLiteStore", func() { // Verify original owner err = sqlStore.ReadTransaction(ctx, func(q *store.Queries) error { - row, err := q.GetPayloadForEntityKey(ctx, key.Bytes()) + row, err := q.GetCurrentPayloadForEntityKey(ctx, key.Bytes()) Expect(err).NotTo(HaveOccurred()) Expect(row.StringAttributes.Values["$owner"]).To(Equal(strings.ToLower(originalOwner.Hex()))) Expect(row.StringAttributes.Values["$creator"]).To(Equal(strings.ToLower(originalOwner.Hex()))) @@ -703,28 +672,22 @@ var _ = Describe("SQLiteStore", func() { // Verify new owner and creator preserved err = sqlStore.ReadTransaction(ctx, func(q *store.Queries) error { - row, err := q.GetPayloadForEntityKey(ctx, key.Bytes()) + row, err := q.GetCurrentPayloadForEntityKey(ctx, key.Bytes()) Expect(err).NotTo(HaveOccurred()) Expect(row.StringAttributes.Values["$owner"]).To(Equal(strings.ToLower(newOwner.Hex()))) // $creator should be preserved Expect(row.StringAttributes.Values["$creator"]).To(Equal(strings.ToLower(originalOwner.Hex()))) // Verify old owner bitmap is removed - oldOwnerBitmap, err := q.EvaluateStringAttributeValueEqual(ctx, store.EvaluateStringAttributeValueEqualParams{ - Name: "$owner", - Value: strings.ToLower(originalOwner.Hex()), - }) - Expect(err).To(HaveOccurred()) + oldOwnerBitmap, err := q.ReconstructLatestStringBitmap(ctx, "$owner", strings.ToLower(originalOwner.Hex())) + Expect(err).NotTo(HaveOccurred()) + Expect(oldOwnerBitmap.IsEmpty()).To(BeTrue()) // Verify new owner bitmap exists - newOwnerBitmap, err := q.EvaluateStringAttributeValueEqual(ctx, store.EvaluateStringAttributeValueEqualParams{ - Name: "$owner", - Value: strings.ToLower(newOwner.Hex()), - }) + newOwnerBitmap, err := q.ReconstructLatestStringBitmap(ctx, "$owner", strings.ToLower(newOwner.Hex())) Expect(err).NotTo(HaveOccurred()) Expect(newOwnerBitmap.ToArray()).To(HaveLen(1)) - _ = oldOwnerBitmap return nil }) Expect(err).NotTo(HaveOccurred()) @@ -815,7 +778,7 @@ var _ = Describe("SQLiteStore", func() { // Verify the update was applied err = sqlStore.ReadTransaction(ctx, func(q *store.Queries) error { - row, err := q.GetPayloadForEntityKey(ctx, key.Bytes()) + row, err := q.GetCurrentPayloadForEntityKey(ctx, key.Bytes()) Expect(err).NotTo(HaveOccurred()) Expect(row.Payload).To(Equal([]byte("final update"))) Expect(row.StringAttributes.Values["status"]).To(Equal("v3")) @@ -926,7 +889,7 @@ var _ = Describe("SQLiteStore", func() { // With `continue operationLoop`, non-last updates are skipped but processing // continues to the next operation. The last update for the key is applied. err = sqlStore.ReadTransaction(ctx, func(q *store.Queries) error { - row, err := q.GetPayloadForEntityKey(ctx, key.Bytes()) + row, err := q.GetCurrentPayloadForEntityKey(ctx, key.Bytes()) Expect(err).NotTo(HaveOccurred()) // The last update (second one) should be applied Expect(row.Payload).To(Equal([]byte("second update - last one"))) @@ -1013,7 +976,7 @@ var _ = Describe("SQLiteStore", func() { // Verify original content is preserved err = sqlStore.ReadTransaction(ctx, func(q *store.Queries) error { - row, err := q.GetPayloadForEntityKey(ctx, key.Bytes()) + row, err := q.GetCurrentPayloadForEntityKey(ctx, key.Bytes()) Expect(err).NotTo(HaveOccurred()) Expect(row.Payload).To(Equal([]byte("original"))) return nil @@ -1077,7 +1040,7 @@ var _ = Describe("SQLiteStore", func() { Expect(err).NotTo(HaveOccurred()) err = sqlStore.ReadTransaction(ctx, func(q *store.Queries) error { - row, err := q.GetPayloadForEntityKey(ctx, key.Bytes()) + row, err := q.GetCurrentPayloadForEntityKey(ctx, key.Bytes()) Expect(err).NotTo(HaveOccurred()) // String attributes diff --git a/store/bitmap_chain.go b/store/bitmap_chain.go new file mode 100644 index 0000000..1bf9825 --- /dev/null +++ b/store/bitmap_chain.go @@ -0,0 +1,212 @@ +package store + +import ( + "context" + "fmt" + "math" + + "github.com/RoaringBitmap/roaring/v2/roaring64" +) + +// KeyframeInterval is the maximum number of delta frames before a new keyframe +// is stored. Analogous to constant C in the bitemporality proposal. +const KeyframeInterval int64 = 128 + +// BitmapChainEntry represents one entry in a keyframe/delta chain. +type BitmapChainEntry struct { + Block uint64 + IsFullBitmap bool + Bitmap *Bitmap +} + +// ReconstructBitmap takes a chain of entries (keyframe first, then deltas in +// block order) and reconstructs the bitmap state at the last entry's block. +// If the chain is empty, returns an empty bitmap. +func ReconstructBitmap(chain []BitmapChainEntry) *Bitmap { + if len(chain) == 0 { + return NewBitmap() + } + + result := NewBitmap() + if chain[0].Bitmap != nil && chain[0].Bitmap.Bitmap != nil { + result.Bitmap = chain[0].Bitmap.Bitmap.Clone() + } + + for _, entry := range chain[1:] { + if entry.Bitmap != nil && entry.Bitmap.Bitmap != nil { + result.Bitmap.Xor(entry.Bitmap.Bitmap) + } + } + + return result +} + +// coalesceBlockToInt64 extracts an int64 from the interface{} returned by +// COALESCE(MAX(block), -1) queries. Returns -1 if the value is nil. +func coalesceBlockToInt64(v interface{}) (int64, error) { + if v == nil { + return -1, nil + } + switch val := v.(type) { + case int64: + return val, nil + case float64: + return int64(val), nil + default: + return -1, fmt.Errorf("unexpected type for block value: %T", v) + } +} + +// ReconstructLatestStringBitmap reconstructs the current (latest) state of a +// string attribute bitmap by loading the full chain from the latest keyframe. +func (q *Queries) ReconstructLatestStringBitmap(ctx context.Context, name, value string) (*Bitmap, error) { + kfBlockRaw, err := q.GetLatestStringKeyframeBlock(ctx, GetLatestStringKeyframeBlockParams{ + Name: name, Value: value, + }) + if err != nil { + return nil, fmt.Errorf("get latest string keyframe block: %w", err) + } + + kfBlock, err := coalesceBlockToInt64(kfBlockRaw) + if err != nil { + return nil, err + } + if kfBlock < 0 { + return NewBitmap(), nil + } + + rows, err := q.GetAllStringBitmapEntriesFrom(ctx, GetAllStringBitmapEntriesFromParams{ + Name: name, Value: value, FromBlock: uint64(kfBlock), + }) + if err != nil { + return nil, fmt.Errorf("get string bitmap entries: %w", err) + } + + chain := make([]BitmapChainEntry, len(rows)) + for i, r := range rows { + chain[i] = BitmapChainEntry{Block: r.Block, IsFullBitmap: r.IsFullBitmap, Bitmap: r.Bitmap} + } + return ReconstructBitmap(chain), nil +} + +// ReconstructLatestNumericBitmap reconstructs the current (latest) state of a +// numeric attribute bitmap. +func (q *Queries) ReconstructLatestNumericBitmap(ctx context.Context, name string, value uint64) (*Bitmap, error) { + kfBlockRaw, err := q.GetLatestNumericKeyframeBlock(ctx, GetLatestNumericKeyframeBlockParams{ + Name: name, Value: value, + }) + if err != nil { + return nil, fmt.Errorf("get latest numeric keyframe block: %w", err) + } + + kfBlock, err := coalesceBlockToInt64(kfBlockRaw) + if err != nil { + return nil, err + } + if kfBlock < 0 { + return NewBitmap(), nil + } + + rows, err := q.GetAllNumericBitmapEntriesFrom(ctx, GetAllNumericBitmapEntriesFromParams{ + Name: name, Value: value, FromBlock: uint64(kfBlock), + }) + if err != nil { + return nil, fmt.Errorf("get numeric bitmap entries: %w", err) + } + + chain := make([]BitmapChainEntry, len(rows)) + for i, r := range rows { + chain[i] = BitmapChainEntry{Block: r.Block, IsFullBitmap: r.IsFullBitmap, Bitmap: r.Bitmap} + } + return ReconstructBitmap(chain), nil +} + +// ReconstructStringBitmapAtBlock reconstructs the string attribute bitmap at a +// specific target block. If block is 0, returns the latest state. +func (q *Queries) ReconstructStringBitmapAtBlock(ctx context.Context, name, value string, block uint64) (*Bitmap, error) { + if block == 0 { + return q.ReconstructLatestStringBitmap(ctx, name, value) + } + + kfBlockRaw, err := q.GetStringKeyframeBlockAtOrBefore(ctx, GetStringKeyframeBlockAtOrBeforeParams{ + Name: name, Value: value, TargetBlock: block, + }) + if err != nil { + return nil, fmt.Errorf("get string keyframe at or before block %d: %w", block, err) + } + + kfBlock, err := coalesceBlockToInt64(kfBlockRaw) + if err != nil { + return nil, err + } + if kfBlock < 0 { + return NewBitmap(), nil + } + + rows, err := q.GetStringBitmapEntriesInRange(ctx, GetStringBitmapEntriesInRangeParams{ + Name: name, Value: value, FromBlock: uint64(kfBlock), ToBlock: block, + }) + if err != nil { + return nil, fmt.Errorf("get string bitmap entries in range: %w", err) + } + + chain := make([]BitmapChainEntry, len(rows)) + for i, r := range rows { + chain[i] = BitmapChainEntry{Block: r.Block, IsFullBitmap: r.IsFullBitmap, Bitmap: r.Bitmap} + } + return ReconstructBitmap(chain), nil +} + +// ReconstructNumericBitmapAtBlock reconstructs the numeric attribute bitmap at +// a specific target block. If block is 0, returns the latest state. +func (q *Queries) ReconstructNumericBitmapAtBlock(ctx context.Context, name string, value, block uint64) (*Bitmap, error) { + if block == 0 { + return q.ReconstructLatestNumericBitmap(ctx, name, value) + } + + kfBlockRaw, err := q.GetNumericKeyframeBlockAtOrBefore(ctx, GetNumericKeyframeBlockAtOrBeforeParams{ + Name: name, Value: value, TargetBlock: block, + }) + if err != nil { + return nil, fmt.Errorf("get numeric keyframe at or before block %d: %w", block, err) + } + + kfBlock, err := coalesceBlockToInt64(kfBlockRaw) + if err != nil { + return nil, err + } + if kfBlock < 0 { + return NewBitmap(), nil + } + + rows, err := q.GetNumericBitmapEntriesInRange(ctx, GetNumericBitmapEntriesInRangeParams{ + Name: name, Value: value, FromBlock: uint64(kfBlock), ToBlock: block, + }) + if err != nil { + return nil, fmt.Errorf("get numeric bitmap entries in range: %w", err) + } + + chain := make([]BitmapChainEntry, len(rows)) + for i, r := range rows { + chain[i] = BitmapChainEntry{Block: r.Block, IsFullBitmap: r.IsFullBitmap, Bitmap: r.Bitmap} + } + return ReconstructBitmap(chain), nil +} + +// EffectiveBlock returns the block value to use in queries. If block is 0 +// (meaning "latest"), returns math.MaxUint64 so that all entries match. +func EffectiveBlock(block uint64) uint64 { + if block == 0 { + return math.MaxUint64 + } + return block +} + +// ComputeDelta computes the XOR of two bitmaps. The result, when XOR'd with +// oldBitmap, produces newBitmap. +func ComputeDelta(oldBitmap, newBitmap *roaring64.Bitmap) *Bitmap { + delta := NewBitmap() + delta.Bitmap = roaring64.Xor(oldBitmap, newBitmap) + delta.RunOptimize() + return delta +} diff --git a/store/db.go b/store/db.go index d164fbe..6aa00d7 100644 --- a/store/db.go +++ b/store/db.go @@ -24,86 +24,137 @@ func New(db DBTX) *Queries { func Prepare(ctx context.Context, db DBTX) (*Queries, error) { q := Queries{db: db} var err error - if q.deleteNumericAttributeValueBitmapStmt, err = db.PrepareContext(ctx, deleteNumericAttributeValueBitmap); err != nil { - return nil, fmt.Errorf("error preparing query DeleteNumericAttributeValueBitmap: %w", err) + if q.closePayloadVersionStmt, err = db.PrepareContext(ctx, closePayloadVersion); err != nil { + return nil, fmt.Errorf("error preparing query ClosePayloadVersion: %w", err) } - if q.deletePayloadForEntityKeyStmt, err = db.PrepareContext(ctx, deletePayloadForEntityKey); err != nil { - return nil, fmt.Errorf("error preparing query DeletePayloadForEntityKey: %w", err) + if q.countDeltasSinceLastKeyframeNumericStmt, err = db.PrepareContext(ctx, countDeltasSinceLastKeyframeNumeric); err != nil { + return nil, fmt.Errorf("error preparing query CountDeltasSinceLastKeyframeNumeric: %w", err) } - if q.deleteStringAttributeValueBitmapStmt, err = db.PrepareContext(ctx, deleteStringAttributeValueBitmap); err != nil { - return nil, fmt.Errorf("error preparing query DeleteStringAttributeValueBitmap: %w", err) + if q.countDeltasSinceLastKeyframeStringStmt, err = db.PrepareContext(ctx, countDeltasSinceLastKeyframeString); err != nil { + return nil, fmt.Errorf("error preparing query CountDeltasSinceLastKeyframeString: %w", err) } - if q.evaluateAllStmt, err = db.PrepareContext(ctx, evaluateAll); err != nil { - return nil, fmt.Errorf("error preparing query EvaluateAll: %w", err) + if q.evaluateAllAtBlockStmt, err = db.PrepareContext(ctx, evaluateAllAtBlock); err != nil { + return nil, fmt.Errorf("error preparing query EvaluateAllAtBlock: %w", err) } - if q.evaluateNumericAttributeValueEqualStmt, err = db.PrepareContext(ctx, evaluateNumericAttributeValueEqual); err != nil { - return nil, fmt.Errorf("error preparing query EvaluateNumericAttributeValueEqual: %w", err) + if q.evaluateAllCurrentStmt, err = db.PrepareContext(ctx, evaluateAllCurrent); err != nil { + return nil, fmt.Errorf("error preparing query EvaluateAllCurrent: %w", err) } - if q.evaluateNumericAttributeValueGreaterOrEqualThanStmt, err = db.PrepareContext(ctx, evaluateNumericAttributeValueGreaterOrEqualThan); err != nil { - return nil, fmt.Errorf("error preparing query EvaluateNumericAttributeValueGreaterOrEqualThan: %w", err) + if q.getAllNumericBitmapEntriesFromStmt, err = db.PrepareContext(ctx, getAllNumericBitmapEntriesFrom); err != nil { + return nil, fmt.Errorf("error preparing query GetAllNumericBitmapEntriesFrom: %w", err) } - if q.evaluateNumericAttributeValueGreaterThanStmt, err = db.PrepareContext(ctx, evaluateNumericAttributeValueGreaterThan); err != nil { - return nil, fmt.Errorf("error preparing query EvaluateNumericAttributeValueGreaterThan: %w", err) + if q.getAllStringBitmapEntriesFromStmt, err = db.PrepareContext(ctx, getAllStringBitmapEntriesFrom); err != nil { + return nil, fmt.Errorf("error preparing query GetAllStringBitmapEntriesFrom: %w", err) } - if q.evaluateNumericAttributeValueInclusionStmt, err = db.PrepareContext(ctx, evaluateNumericAttributeValueInclusion); err != nil { - return nil, fmt.Errorf("error preparing query EvaluateNumericAttributeValueInclusion: %w", err) + if q.getCurrentPayloadForEntityKeyStmt, err = db.PrepareContext(ctx, getCurrentPayloadForEntityKey); err != nil { + return nil, fmt.Errorf("error preparing query GetCurrentPayloadForEntityKey: %w", err) } - if q.evaluateNumericAttributeValueLessOrEqualThanStmt, err = db.PrepareContext(ctx, evaluateNumericAttributeValueLessOrEqualThan); err != nil { - return nil, fmt.Errorf("error preparing query EvaluateNumericAttributeValueLessOrEqualThan: %w", err) + if q.getLastBlockStmt, err = db.PrepareContext(ctx, getLastBlock); err != nil { + return nil, fmt.Errorf("error preparing query GetLastBlock: %w", err) } - if q.evaluateNumericAttributeValueLowerThanStmt, err = db.PrepareContext(ctx, evaluateNumericAttributeValueLowerThan); err != nil { - return nil, fmt.Errorf("error preparing query EvaluateNumericAttributeValueLowerThan: %w", err) + if q.getLatestNumericKeyframeBlockStmt, err = db.PrepareContext(ctx, getLatestNumericKeyframeBlock); err != nil { + return nil, fmt.Errorf("error preparing query GetLatestNumericKeyframeBlock: %w", err) } - if q.evaluateNumericAttributeValueNotEqualStmt, err = db.PrepareContext(ctx, evaluateNumericAttributeValueNotEqual); err != nil { - return nil, fmt.Errorf("error preparing query EvaluateNumericAttributeValueNotEqual: %w", err) + if q.getLatestStringKeyframeBlockStmt, err = db.PrepareContext(ctx, getLatestStringKeyframeBlock); err != nil { + return nil, fmt.Errorf("error preparing query GetLatestStringKeyframeBlock: %w", err) } - if q.evaluateNumericAttributeValueNotInclusionStmt, err = db.PrepareContext(ctx, evaluateNumericAttributeValueNotInclusion); err != nil { - return nil, fmt.Errorf("error preparing query EvaluateNumericAttributeValueNotInclusion: %w", err) + if q.getMatchingNumericValuesEqualStmt, err = db.PrepareContext(ctx, getMatchingNumericValuesEqual); err != nil { + return nil, fmt.Errorf("error preparing query GetMatchingNumericValuesEqual: %w", err) } - if q.evaluateStringAttributeValueEqualStmt, err = db.PrepareContext(ctx, evaluateStringAttributeValueEqual); err != nil { - return nil, fmt.Errorf("error preparing query EvaluateStringAttributeValueEqual: %w", err) + if q.getMatchingNumericValuesGreaterOrEqualThanStmt, err = db.PrepareContext(ctx, getMatchingNumericValuesGreaterOrEqualThan); err != nil { + return nil, fmt.Errorf("error preparing query GetMatchingNumericValuesGreaterOrEqualThan: %w", err) } - if q.evaluateStringAttributeValueGlobStmt, err = db.PrepareContext(ctx, evaluateStringAttributeValueGlob); err != nil { - return nil, fmt.Errorf("error preparing query EvaluateStringAttributeValueGlob: %w", err) + if q.getMatchingNumericValuesGreaterThanStmt, err = db.PrepareContext(ctx, getMatchingNumericValuesGreaterThan); err != nil { + return nil, fmt.Errorf("error preparing query GetMatchingNumericValuesGreaterThan: %w", err) } - if q.evaluateStringAttributeValueGreaterOrEqualThanStmt, err = db.PrepareContext(ctx, evaluateStringAttributeValueGreaterOrEqualThan); err != nil { - return nil, fmt.Errorf("error preparing query EvaluateStringAttributeValueGreaterOrEqualThan: %w", err) + if q.getMatchingNumericValuesInclusionStmt, err = db.PrepareContext(ctx, getMatchingNumericValuesInclusion); err != nil { + return nil, fmt.Errorf("error preparing query GetMatchingNumericValuesInclusion: %w", err) } - if q.evaluateStringAttributeValueGreaterThanStmt, err = db.PrepareContext(ctx, evaluateStringAttributeValueGreaterThan); err != nil { - return nil, fmt.Errorf("error preparing query EvaluateStringAttributeValueGreaterThan: %w", err) + if q.getMatchingNumericValuesLessOrEqualThanStmt, err = db.PrepareContext(ctx, getMatchingNumericValuesLessOrEqualThan); err != nil { + return nil, fmt.Errorf("error preparing query GetMatchingNumericValuesLessOrEqualThan: %w", err) } - if q.evaluateStringAttributeValueInclusionStmt, err = db.PrepareContext(ctx, evaluateStringAttributeValueInclusion); err != nil { - return nil, fmt.Errorf("error preparing query EvaluateStringAttributeValueInclusion: %w", err) + if q.getMatchingNumericValuesLessThanStmt, err = db.PrepareContext(ctx, getMatchingNumericValuesLessThan); err != nil { + return nil, fmt.Errorf("error preparing query GetMatchingNumericValuesLessThan: %w", err) } - if q.evaluateStringAttributeValueLessOrEqualThanStmt, err = db.PrepareContext(ctx, evaluateStringAttributeValueLessOrEqualThan); err != nil { - return nil, fmt.Errorf("error preparing query EvaluateStringAttributeValueLessOrEqualThan: %w", err) + if q.getMatchingNumericValuesNotEqualStmt, err = db.PrepareContext(ctx, getMatchingNumericValuesNotEqual); err != nil { + return nil, fmt.Errorf("error preparing query GetMatchingNumericValuesNotEqual: %w", err) } - if q.evaluateStringAttributeValueLowerThanStmt, err = db.PrepareContext(ctx, evaluateStringAttributeValueLowerThan); err != nil { - return nil, fmt.Errorf("error preparing query EvaluateStringAttributeValueLowerThan: %w", err) + if q.getMatchingNumericValuesNotInclusionStmt, err = db.PrepareContext(ctx, getMatchingNumericValuesNotInclusion); err != nil { + return nil, fmt.Errorf("error preparing query GetMatchingNumericValuesNotInclusion: %w", err) } - if q.evaluateStringAttributeValueNotEqualStmt, err = db.PrepareContext(ctx, evaluateStringAttributeValueNotEqual); err != nil { - return nil, fmt.Errorf("error preparing query EvaluateStringAttributeValueNotEqual: %w", err) + if q.getMatchingStringValuesEqualStmt, err = db.PrepareContext(ctx, getMatchingStringValuesEqual); err != nil { + return nil, fmt.Errorf("error preparing query GetMatchingStringValuesEqual: %w", err) } - if q.evaluateStringAttributeValueNotGlobStmt, err = db.PrepareContext(ctx, evaluateStringAttributeValueNotGlob); err != nil { - return nil, fmt.Errorf("error preparing query EvaluateStringAttributeValueNotGlob: %w", err) + if q.getMatchingStringValuesGlobStmt, err = db.PrepareContext(ctx, getMatchingStringValuesGlob); err != nil { + return nil, fmt.Errorf("error preparing query GetMatchingStringValuesGlob: %w", err) } - if q.evaluateStringAttributeValueNotInclusionStmt, err = db.PrepareContext(ctx, evaluateStringAttributeValueNotInclusion); err != nil { - return nil, fmt.Errorf("error preparing query EvaluateStringAttributeValueNotInclusion: %w", err) + if q.getMatchingStringValuesGreaterOrEqualThanStmt, err = db.PrepareContext(ctx, getMatchingStringValuesGreaterOrEqualThan); err != nil { + return nil, fmt.Errorf("error preparing query GetMatchingStringValuesGreaterOrEqualThan: %w", err) } - if q.getLastBlockStmt, err = db.PrepareContext(ctx, getLastBlock); err != nil { - return nil, fmt.Errorf("error preparing query GetLastBlock: %w", err) + if q.getMatchingStringValuesGreaterThanStmt, err = db.PrepareContext(ctx, getMatchingStringValuesGreaterThan); err != nil { + return nil, fmt.Errorf("error preparing query GetMatchingStringValuesGreaterThan: %w", err) + } + if q.getMatchingStringValuesInclusionStmt, err = db.PrepareContext(ctx, getMatchingStringValuesInclusion); err != nil { + return nil, fmt.Errorf("error preparing query GetMatchingStringValuesInclusion: %w", err) + } + if q.getMatchingStringValuesLessOrEqualThanStmt, err = db.PrepareContext(ctx, getMatchingStringValuesLessOrEqualThan); err != nil { + return nil, fmt.Errorf("error preparing query GetMatchingStringValuesLessOrEqualThan: %w", err) + } + if q.getMatchingStringValuesLessThanStmt, err = db.PrepareContext(ctx, getMatchingStringValuesLessThan); err != nil { + return nil, fmt.Errorf("error preparing query GetMatchingStringValuesLessThan: %w", err) + } + if q.getMatchingStringValuesNotEqualStmt, err = db.PrepareContext(ctx, getMatchingStringValuesNotEqual); err != nil { + return nil, fmt.Errorf("error preparing query GetMatchingStringValuesNotEqual: %w", err) + } + if q.getMatchingStringValuesNotGlobStmt, err = db.PrepareContext(ctx, getMatchingStringValuesNotGlob); err != nil { + return nil, fmt.Errorf("error preparing query GetMatchingStringValuesNotGlob: %w", err) + } + if q.getMatchingStringValuesNotInclusionStmt, err = db.PrepareContext(ctx, getMatchingStringValuesNotInclusion); err != nil { + return nil, fmt.Errorf("error preparing query GetMatchingStringValuesNotInclusion: %w", err) } if q.getNumberOfEntitiesStmt, err = db.PrepareContext(ctx, getNumberOfEntities); err != nil { return nil, fmt.Errorf("error preparing query GetNumberOfEntities: %w", err) } - if q.getNumericAttributeValueBitmapStmt, err = db.PrepareContext(ctx, getNumericAttributeValueBitmap); err != nil { - return nil, fmt.Errorf("error preparing query GetNumericAttributeValueBitmap: %w", err) + if q.getNumericBitmapEntriesInRangeStmt, err = db.PrepareContext(ctx, getNumericBitmapEntriesInRange); err != nil { + return nil, fmt.Errorf("error preparing query GetNumericBitmapEntriesInRange: %w", err) + } + if q.getNumericKeyframeBlockAtOrBeforeStmt, err = db.PrepareContext(ctx, getNumericKeyframeBlockAtOrBefore); err != nil { + return nil, fmt.Errorf("error preparing query GetNumericKeyframeBlockAtOrBefore: %w", err) + } + if q.getStringBitmapEntriesInRangeStmt, err = db.PrepareContext(ctx, getStringBitmapEntriesInRange); err != nil { + return nil, fmt.Errorf("error preparing query GetStringBitmapEntriesInRange: %w", err) + } + if q.getStringKeyframeBlockAtOrBeforeStmt, err = db.PrepareContext(ctx, getStringKeyframeBlockAtOrBefore); err != nil { + return nil, fmt.Errorf("error preparing query GetStringKeyframeBlockAtOrBefore: %w", err) + } + if q.insertNumericBitmapEntryStmt, err = db.PrepareContext(ctx, insertNumericBitmapEntry); err != nil { + return nil, fmt.Errorf("error preparing query InsertNumericBitmapEntry: %w", err) + } + if q.insertPayloadStmt, err = db.PrepareContext(ctx, insertPayload); err != nil { + return nil, fmt.Errorf("error preparing query InsertPayload: %w", err) + } + if q.insertStringBitmapEntryStmt, err = db.PrepareContext(ctx, insertStringBitmapEntry); err != nil { + return nil, fmt.Errorf("error preparing query InsertStringBitmapEntry: %w", err) + } + if q.pruneNumericBitmapsBeforeStmt, err = db.PrepareContext(ctx, pruneNumericBitmapsBefore); err != nil { + return nil, fmt.Errorf("error preparing query PruneNumericBitmapsBefore: %w", err) + } + if q.prunePayloadsBeforeStmt, err = db.PrepareContext(ctx, prunePayloadsBefore); err != nil { + return nil, fmt.Errorf("error preparing query PrunePayloadsBefore: %w", err) } - if q.getPayloadForEntityKeyStmt, err = db.PrepareContext(ctx, getPayloadForEntityKey); err != nil { - return nil, fmt.Errorf("error preparing query GetPayloadForEntityKey: %w", err) + if q.pruneStringBitmapsBeforeStmt, err = db.PrepareContext(ctx, pruneStringBitmapsBefore); err != nil { + return nil, fmt.Errorf("error preparing query PruneStringBitmapsBefore: %w", err) } - if q.getStringAttributeValueBitmapStmt, err = db.PrepareContext(ctx, getStringAttributeValueBitmap); err != nil { - return nil, fmt.Errorf("error preparing query GetStringAttributeValueBitmap: %w", err) + if q.reorgDeleteNewPayloadsStmt, err = db.PrepareContext(ctx, reorgDeleteNewPayloads); err != nil { + return nil, fmt.Errorf("error preparing query ReorgDeleteNewPayloads: %w", err) + } + if q.reorgDeleteNumericBitmapsStmt, err = db.PrepareContext(ctx, reorgDeleteNumericBitmaps); err != nil { + return nil, fmt.Errorf("error preparing query ReorgDeleteNumericBitmaps: %w", err) + } + if q.reorgDeleteStringBitmapsStmt, err = db.PrepareContext(ctx, reorgDeleteStringBitmaps); err != nil { + return nil, fmt.Errorf("error preparing query ReorgDeleteStringBitmaps: %w", err) + } + if q.reorgReopenPayloadsStmt, err = db.PrepareContext(ctx, reorgReopenPayloads); err != nil { + return nil, fmt.Errorf("error preparing query ReorgReopenPayloads: %w", err) } if q.retrievePayloadsStmt, err = db.PrepareContext(ctx, retrievePayloads); err != nil { return nil, fmt.Errorf("error preparing query RetrievePayloads: %w", err) @@ -111,133 +162,154 @@ func Prepare(ctx context.Context, db DBTX) (*Queries, error) { if q.upsertLastBlockStmt, err = db.PrepareContext(ctx, upsertLastBlock); err != nil { return nil, fmt.Errorf("error preparing query UpsertLastBlock: %w", err) } - if q.upsertNumericAttributeValueBitmapStmt, err = db.PrepareContext(ctx, upsertNumericAttributeValueBitmap); err != nil { - return nil, fmt.Errorf("error preparing query UpsertNumericAttributeValueBitmap: %w", err) - } - if q.upsertPayloadStmt, err = db.PrepareContext(ctx, upsertPayload); err != nil { - return nil, fmt.Errorf("error preparing query UpsertPayload: %w", err) - } - if q.upsertStringAttributeValueBitmapStmt, err = db.PrepareContext(ctx, upsertStringAttributeValueBitmap); err != nil { - return nil, fmt.Errorf("error preparing query UpsertStringAttributeValueBitmap: %w", err) - } return &q, nil } func (q *Queries) Close() error { var err error - if q.deleteNumericAttributeValueBitmapStmt != nil { - if cerr := q.deleteNumericAttributeValueBitmapStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing deleteNumericAttributeValueBitmapStmt: %w", cerr) + if q.closePayloadVersionStmt != nil { + if cerr := q.closePayloadVersionStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing closePayloadVersionStmt: %w", cerr) } } - if q.deletePayloadForEntityKeyStmt != nil { - if cerr := q.deletePayloadForEntityKeyStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing deletePayloadForEntityKeyStmt: %w", cerr) + if q.countDeltasSinceLastKeyframeNumericStmt != nil { + if cerr := q.countDeltasSinceLastKeyframeNumericStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing countDeltasSinceLastKeyframeNumericStmt: %w", cerr) } } - if q.deleteStringAttributeValueBitmapStmt != nil { - if cerr := q.deleteStringAttributeValueBitmapStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing deleteStringAttributeValueBitmapStmt: %w", cerr) + if q.countDeltasSinceLastKeyframeStringStmt != nil { + if cerr := q.countDeltasSinceLastKeyframeStringStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing countDeltasSinceLastKeyframeStringStmt: %w", cerr) } } - if q.evaluateAllStmt != nil { - if cerr := q.evaluateAllStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing evaluateAllStmt: %w", cerr) + if q.evaluateAllAtBlockStmt != nil { + if cerr := q.evaluateAllAtBlockStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing evaluateAllAtBlockStmt: %w", cerr) } } - if q.evaluateNumericAttributeValueEqualStmt != nil { - if cerr := q.evaluateNumericAttributeValueEqualStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing evaluateNumericAttributeValueEqualStmt: %w", cerr) + if q.evaluateAllCurrentStmt != nil { + if cerr := q.evaluateAllCurrentStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing evaluateAllCurrentStmt: %w", cerr) } } - if q.evaluateNumericAttributeValueGreaterOrEqualThanStmt != nil { - if cerr := q.evaluateNumericAttributeValueGreaterOrEqualThanStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing evaluateNumericAttributeValueGreaterOrEqualThanStmt: %w", cerr) + if q.getAllNumericBitmapEntriesFromStmt != nil { + if cerr := q.getAllNumericBitmapEntriesFromStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getAllNumericBitmapEntriesFromStmt: %w", cerr) } } - if q.evaluateNumericAttributeValueGreaterThanStmt != nil { - if cerr := q.evaluateNumericAttributeValueGreaterThanStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing evaluateNumericAttributeValueGreaterThanStmt: %w", cerr) + if q.getAllStringBitmapEntriesFromStmt != nil { + if cerr := q.getAllStringBitmapEntriesFromStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getAllStringBitmapEntriesFromStmt: %w", cerr) } } - if q.evaluateNumericAttributeValueInclusionStmt != nil { - if cerr := q.evaluateNumericAttributeValueInclusionStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing evaluateNumericAttributeValueInclusionStmt: %w", cerr) + if q.getCurrentPayloadForEntityKeyStmt != nil { + if cerr := q.getCurrentPayloadForEntityKeyStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getCurrentPayloadForEntityKeyStmt: %w", cerr) } } - if q.evaluateNumericAttributeValueLessOrEqualThanStmt != nil { - if cerr := q.evaluateNumericAttributeValueLessOrEqualThanStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing evaluateNumericAttributeValueLessOrEqualThanStmt: %w", cerr) + if q.getLastBlockStmt != nil { + if cerr := q.getLastBlockStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getLastBlockStmt: %w", cerr) } } - if q.evaluateNumericAttributeValueLowerThanStmt != nil { - if cerr := q.evaluateNumericAttributeValueLowerThanStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing evaluateNumericAttributeValueLowerThanStmt: %w", cerr) + if q.getLatestNumericKeyframeBlockStmt != nil { + if cerr := q.getLatestNumericKeyframeBlockStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getLatestNumericKeyframeBlockStmt: %w", cerr) } } - if q.evaluateNumericAttributeValueNotEqualStmt != nil { - if cerr := q.evaluateNumericAttributeValueNotEqualStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing evaluateNumericAttributeValueNotEqualStmt: %w", cerr) + if q.getLatestStringKeyframeBlockStmt != nil { + if cerr := q.getLatestStringKeyframeBlockStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getLatestStringKeyframeBlockStmt: %w", cerr) } } - if q.evaluateNumericAttributeValueNotInclusionStmt != nil { - if cerr := q.evaluateNumericAttributeValueNotInclusionStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing evaluateNumericAttributeValueNotInclusionStmt: %w", cerr) + if q.getMatchingNumericValuesEqualStmt != nil { + if cerr := q.getMatchingNumericValuesEqualStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getMatchingNumericValuesEqualStmt: %w", cerr) } } - if q.evaluateStringAttributeValueEqualStmt != nil { - if cerr := q.evaluateStringAttributeValueEqualStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing evaluateStringAttributeValueEqualStmt: %w", cerr) + if q.getMatchingNumericValuesGreaterOrEqualThanStmt != nil { + if cerr := q.getMatchingNumericValuesGreaterOrEqualThanStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getMatchingNumericValuesGreaterOrEqualThanStmt: %w", cerr) } } - if q.evaluateStringAttributeValueGlobStmt != nil { - if cerr := q.evaluateStringAttributeValueGlobStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing evaluateStringAttributeValueGlobStmt: %w", cerr) + if q.getMatchingNumericValuesGreaterThanStmt != nil { + if cerr := q.getMatchingNumericValuesGreaterThanStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getMatchingNumericValuesGreaterThanStmt: %w", cerr) } } - if q.evaluateStringAttributeValueGreaterOrEqualThanStmt != nil { - if cerr := q.evaluateStringAttributeValueGreaterOrEqualThanStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing evaluateStringAttributeValueGreaterOrEqualThanStmt: %w", cerr) + if q.getMatchingNumericValuesInclusionStmt != nil { + if cerr := q.getMatchingNumericValuesInclusionStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getMatchingNumericValuesInclusionStmt: %w", cerr) } } - if q.evaluateStringAttributeValueGreaterThanStmt != nil { - if cerr := q.evaluateStringAttributeValueGreaterThanStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing evaluateStringAttributeValueGreaterThanStmt: %w", cerr) + if q.getMatchingNumericValuesLessOrEqualThanStmt != nil { + if cerr := q.getMatchingNumericValuesLessOrEqualThanStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getMatchingNumericValuesLessOrEqualThanStmt: %w", cerr) } } - if q.evaluateStringAttributeValueInclusionStmt != nil { - if cerr := q.evaluateStringAttributeValueInclusionStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing evaluateStringAttributeValueInclusionStmt: %w", cerr) + if q.getMatchingNumericValuesLessThanStmt != nil { + if cerr := q.getMatchingNumericValuesLessThanStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getMatchingNumericValuesLessThanStmt: %w", cerr) } } - if q.evaluateStringAttributeValueLessOrEqualThanStmt != nil { - if cerr := q.evaluateStringAttributeValueLessOrEqualThanStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing evaluateStringAttributeValueLessOrEqualThanStmt: %w", cerr) + if q.getMatchingNumericValuesNotEqualStmt != nil { + if cerr := q.getMatchingNumericValuesNotEqualStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getMatchingNumericValuesNotEqualStmt: %w", cerr) } } - if q.evaluateStringAttributeValueLowerThanStmt != nil { - if cerr := q.evaluateStringAttributeValueLowerThanStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing evaluateStringAttributeValueLowerThanStmt: %w", cerr) + if q.getMatchingNumericValuesNotInclusionStmt != nil { + if cerr := q.getMatchingNumericValuesNotInclusionStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getMatchingNumericValuesNotInclusionStmt: %w", cerr) } } - if q.evaluateStringAttributeValueNotEqualStmt != nil { - if cerr := q.evaluateStringAttributeValueNotEqualStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing evaluateStringAttributeValueNotEqualStmt: %w", cerr) + if q.getMatchingStringValuesEqualStmt != nil { + if cerr := q.getMatchingStringValuesEqualStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getMatchingStringValuesEqualStmt: %w", cerr) } } - if q.evaluateStringAttributeValueNotGlobStmt != nil { - if cerr := q.evaluateStringAttributeValueNotGlobStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing evaluateStringAttributeValueNotGlobStmt: %w", cerr) + if q.getMatchingStringValuesGlobStmt != nil { + if cerr := q.getMatchingStringValuesGlobStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getMatchingStringValuesGlobStmt: %w", cerr) } } - if q.evaluateStringAttributeValueNotInclusionStmt != nil { - if cerr := q.evaluateStringAttributeValueNotInclusionStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing evaluateStringAttributeValueNotInclusionStmt: %w", cerr) + if q.getMatchingStringValuesGreaterOrEqualThanStmt != nil { + if cerr := q.getMatchingStringValuesGreaterOrEqualThanStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getMatchingStringValuesGreaterOrEqualThanStmt: %w", cerr) } } - if q.getLastBlockStmt != nil { - if cerr := q.getLastBlockStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing getLastBlockStmt: %w", cerr) + if q.getMatchingStringValuesGreaterThanStmt != nil { + if cerr := q.getMatchingStringValuesGreaterThanStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getMatchingStringValuesGreaterThanStmt: %w", cerr) + } + } + if q.getMatchingStringValuesInclusionStmt != nil { + if cerr := q.getMatchingStringValuesInclusionStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getMatchingStringValuesInclusionStmt: %w", cerr) + } + } + if q.getMatchingStringValuesLessOrEqualThanStmt != nil { + if cerr := q.getMatchingStringValuesLessOrEqualThanStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getMatchingStringValuesLessOrEqualThanStmt: %w", cerr) + } + } + if q.getMatchingStringValuesLessThanStmt != nil { + if cerr := q.getMatchingStringValuesLessThanStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getMatchingStringValuesLessThanStmt: %w", cerr) + } + } + if q.getMatchingStringValuesNotEqualStmt != nil { + if cerr := q.getMatchingStringValuesNotEqualStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getMatchingStringValuesNotEqualStmt: %w", cerr) + } + } + if q.getMatchingStringValuesNotGlobStmt != nil { + if cerr := q.getMatchingStringValuesNotGlobStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getMatchingStringValuesNotGlobStmt: %w", cerr) + } + } + if q.getMatchingStringValuesNotInclusionStmt != nil { + if cerr := q.getMatchingStringValuesNotInclusionStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getMatchingStringValuesNotInclusionStmt: %w", cerr) } } if q.getNumberOfEntitiesStmt != nil { @@ -245,44 +317,84 @@ func (q *Queries) Close() error { err = fmt.Errorf("error closing getNumberOfEntitiesStmt: %w", cerr) } } - if q.getNumericAttributeValueBitmapStmt != nil { - if cerr := q.getNumericAttributeValueBitmapStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing getNumericAttributeValueBitmapStmt: %w", cerr) + if q.getNumericBitmapEntriesInRangeStmt != nil { + if cerr := q.getNumericBitmapEntriesInRangeStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getNumericBitmapEntriesInRangeStmt: %w", cerr) } } - if q.getPayloadForEntityKeyStmt != nil { - if cerr := q.getPayloadForEntityKeyStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing getPayloadForEntityKeyStmt: %w", cerr) + if q.getNumericKeyframeBlockAtOrBeforeStmt != nil { + if cerr := q.getNumericKeyframeBlockAtOrBeforeStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getNumericKeyframeBlockAtOrBeforeStmt: %w", cerr) } } - if q.getStringAttributeValueBitmapStmt != nil { - if cerr := q.getStringAttributeValueBitmapStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing getStringAttributeValueBitmapStmt: %w", cerr) + if q.getStringBitmapEntriesInRangeStmt != nil { + if cerr := q.getStringBitmapEntriesInRangeStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getStringBitmapEntriesInRangeStmt: %w", cerr) } } - if q.retrievePayloadsStmt != nil { - if cerr := q.retrievePayloadsStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing retrievePayloadsStmt: %w", cerr) + if q.getStringKeyframeBlockAtOrBeforeStmt != nil { + if cerr := q.getStringKeyframeBlockAtOrBeforeStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getStringKeyframeBlockAtOrBeforeStmt: %w", cerr) } } - if q.upsertLastBlockStmt != nil { - if cerr := q.upsertLastBlockStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing upsertLastBlockStmt: %w", cerr) + if q.insertNumericBitmapEntryStmt != nil { + if cerr := q.insertNumericBitmapEntryStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing insertNumericBitmapEntryStmt: %w", cerr) } } - if q.upsertNumericAttributeValueBitmapStmt != nil { - if cerr := q.upsertNumericAttributeValueBitmapStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing upsertNumericAttributeValueBitmapStmt: %w", cerr) + if q.insertPayloadStmt != nil { + if cerr := q.insertPayloadStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing insertPayloadStmt: %w", cerr) } } - if q.upsertPayloadStmt != nil { - if cerr := q.upsertPayloadStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing upsertPayloadStmt: %w", cerr) + if q.insertStringBitmapEntryStmt != nil { + if cerr := q.insertStringBitmapEntryStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing insertStringBitmapEntryStmt: %w", cerr) } } - if q.upsertStringAttributeValueBitmapStmt != nil { - if cerr := q.upsertStringAttributeValueBitmapStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing upsertStringAttributeValueBitmapStmt: %w", cerr) + if q.pruneNumericBitmapsBeforeStmt != nil { + if cerr := q.pruneNumericBitmapsBeforeStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing pruneNumericBitmapsBeforeStmt: %w", cerr) + } + } + if q.prunePayloadsBeforeStmt != nil { + if cerr := q.prunePayloadsBeforeStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing prunePayloadsBeforeStmt: %w", cerr) + } + } + if q.pruneStringBitmapsBeforeStmt != nil { + if cerr := q.pruneStringBitmapsBeforeStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing pruneStringBitmapsBeforeStmt: %w", cerr) + } + } + if q.reorgDeleteNewPayloadsStmt != nil { + if cerr := q.reorgDeleteNewPayloadsStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing reorgDeleteNewPayloadsStmt: %w", cerr) + } + } + if q.reorgDeleteNumericBitmapsStmt != nil { + if cerr := q.reorgDeleteNumericBitmapsStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing reorgDeleteNumericBitmapsStmt: %w", cerr) + } + } + if q.reorgDeleteStringBitmapsStmt != nil { + if cerr := q.reorgDeleteStringBitmapsStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing reorgDeleteStringBitmapsStmt: %w", cerr) + } + } + if q.reorgReopenPayloadsStmt != nil { + if cerr := q.reorgReopenPayloadsStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing reorgReopenPayloadsStmt: %w", cerr) + } + } + if q.retrievePayloadsStmt != nil { + if cerr := q.retrievePayloadsStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing retrievePayloadsStmt: %w", cerr) + } + } + if q.upsertLastBlockStmt != nil { + if cerr := q.upsertLastBlockStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing upsertLastBlockStmt: %w", cerr) } } return err @@ -322,77 +434,105 @@ func (q *Queries) queryRow(ctx context.Context, stmt *sql.Stmt, query string, ar } type Queries struct { - db DBTX - tx *sql.Tx - deleteNumericAttributeValueBitmapStmt *sql.Stmt - deletePayloadForEntityKeyStmt *sql.Stmt - deleteStringAttributeValueBitmapStmt *sql.Stmt - evaluateAllStmt *sql.Stmt - evaluateNumericAttributeValueEqualStmt *sql.Stmt - evaluateNumericAttributeValueGreaterOrEqualThanStmt *sql.Stmt - evaluateNumericAttributeValueGreaterThanStmt *sql.Stmt - evaluateNumericAttributeValueInclusionStmt *sql.Stmt - evaluateNumericAttributeValueLessOrEqualThanStmt *sql.Stmt - evaluateNumericAttributeValueLowerThanStmt *sql.Stmt - evaluateNumericAttributeValueNotEqualStmt *sql.Stmt - evaluateNumericAttributeValueNotInclusionStmt *sql.Stmt - evaluateStringAttributeValueEqualStmt *sql.Stmt - evaluateStringAttributeValueGlobStmt *sql.Stmt - evaluateStringAttributeValueGreaterOrEqualThanStmt *sql.Stmt - evaluateStringAttributeValueGreaterThanStmt *sql.Stmt - evaluateStringAttributeValueInclusionStmt *sql.Stmt - evaluateStringAttributeValueLessOrEqualThanStmt *sql.Stmt - evaluateStringAttributeValueLowerThanStmt *sql.Stmt - evaluateStringAttributeValueNotEqualStmt *sql.Stmt - evaluateStringAttributeValueNotGlobStmt *sql.Stmt - evaluateStringAttributeValueNotInclusionStmt *sql.Stmt - getLastBlockStmt *sql.Stmt - getNumberOfEntitiesStmt *sql.Stmt - getNumericAttributeValueBitmapStmt *sql.Stmt - getPayloadForEntityKeyStmt *sql.Stmt - getStringAttributeValueBitmapStmt *sql.Stmt - retrievePayloadsStmt *sql.Stmt - upsertLastBlockStmt *sql.Stmt - upsertNumericAttributeValueBitmapStmt *sql.Stmt - upsertPayloadStmt *sql.Stmt - upsertStringAttributeValueBitmapStmt *sql.Stmt + db DBTX + tx *sql.Tx + closePayloadVersionStmt *sql.Stmt + countDeltasSinceLastKeyframeNumericStmt *sql.Stmt + countDeltasSinceLastKeyframeStringStmt *sql.Stmt + evaluateAllAtBlockStmt *sql.Stmt + evaluateAllCurrentStmt *sql.Stmt + getAllNumericBitmapEntriesFromStmt *sql.Stmt + getAllStringBitmapEntriesFromStmt *sql.Stmt + getCurrentPayloadForEntityKeyStmt *sql.Stmt + getLastBlockStmt *sql.Stmt + getLatestNumericKeyframeBlockStmt *sql.Stmt + getLatestStringKeyframeBlockStmt *sql.Stmt + getMatchingNumericValuesEqualStmt *sql.Stmt + getMatchingNumericValuesGreaterOrEqualThanStmt *sql.Stmt + getMatchingNumericValuesGreaterThanStmt *sql.Stmt + getMatchingNumericValuesInclusionStmt *sql.Stmt + getMatchingNumericValuesLessOrEqualThanStmt *sql.Stmt + getMatchingNumericValuesLessThanStmt *sql.Stmt + getMatchingNumericValuesNotEqualStmt *sql.Stmt + getMatchingNumericValuesNotInclusionStmt *sql.Stmt + getMatchingStringValuesEqualStmt *sql.Stmt + getMatchingStringValuesGlobStmt *sql.Stmt + getMatchingStringValuesGreaterOrEqualThanStmt *sql.Stmt + getMatchingStringValuesGreaterThanStmt *sql.Stmt + getMatchingStringValuesInclusionStmt *sql.Stmt + getMatchingStringValuesLessOrEqualThanStmt *sql.Stmt + getMatchingStringValuesLessThanStmt *sql.Stmt + getMatchingStringValuesNotEqualStmt *sql.Stmt + getMatchingStringValuesNotGlobStmt *sql.Stmt + getMatchingStringValuesNotInclusionStmt *sql.Stmt + getNumberOfEntitiesStmt *sql.Stmt + getNumericBitmapEntriesInRangeStmt *sql.Stmt + getNumericKeyframeBlockAtOrBeforeStmt *sql.Stmt + getStringBitmapEntriesInRangeStmt *sql.Stmt + getStringKeyframeBlockAtOrBeforeStmt *sql.Stmt + insertNumericBitmapEntryStmt *sql.Stmt + insertPayloadStmt *sql.Stmt + insertStringBitmapEntryStmt *sql.Stmt + pruneNumericBitmapsBeforeStmt *sql.Stmt + prunePayloadsBeforeStmt *sql.Stmt + pruneStringBitmapsBeforeStmt *sql.Stmt + reorgDeleteNewPayloadsStmt *sql.Stmt + reorgDeleteNumericBitmapsStmt *sql.Stmt + reorgDeleteStringBitmapsStmt *sql.Stmt + reorgReopenPayloadsStmt *sql.Stmt + retrievePayloadsStmt *sql.Stmt + upsertLastBlockStmt *sql.Stmt } func (q *Queries) WithTx(tx *sql.Tx) *Queries { return &Queries{ - db: tx, - tx: tx, - deleteNumericAttributeValueBitmapStmt: q.deleteNumericAttributeValueBitmapStmt, - deletePayloadForEntityKeyStmt: q.deletePayloadForEntityKeyStmt, - deleteStringAttributeValueBitmapStmt: q.deleteStringAttributeValueBitmapStmt, - evaluateAllStmt: q.evaluateAllStmt, - evaluateNumericAttributeValueEqualStmt: q.evaluateNumericAttributeValueEqualStmt, - evaluateNumericAttributeValueGreaterOrEqualThanStmt: q.evaluateNumericAttributeValueGreaterOrEqualThanStmt, - evaluateNumericAttributeValueGreaterThanStmt: q.evaluateNumericAttributeValueGreaterThanStmt, - evaluateNumericAttributeValueInclusionStmt: q.evaluateNumericAttributeValueInclusionStmt, - evaluateNumericAttributeValueLessOrEqualThanStmt: q.evaluateNumericAttributeValueLessOrEqualThanStmt, - evaluateNumericAttributeValueLowerThanStmt: q.evaluateNumericAttributeValueLowerThanStmt, - evaluateNumericAttributeValueNotEqualStmt: q.evaluateNumericAttributeValueNotEqualStmt, - evaluateNumericAttributeValueNotInclusionStmt: q.evaluateNumericAttributeValueNotInclusionStmt, - evaluateStringAttributeValueEqualStmt: q.evaluateStringAttributeValueEqualStmt, - evaluateStringAttributeValueGlobStmt: q.evaluateStringAttributeValueGlobStmt, - evaluateStringAttributeValueGreaterOrEqualThanStmt: q.evaluateStringAttributeValueGreaterOrEqualThanStmt, - evaluateStringAttributeValueGreaterThanStmt: q.evaluateStringAttributeValueGreaterThanStmt, - evaluateStringAttributeValueInclusionStmt: q.evaluateStringAttributeValueInclusionStmt, - evaluateStringAttributeValueLessOrEqualThanStmt: q.evaluateStringAttributeValueLessOrEqualThanStmt, - evaluateStringAttributeValueLowerThanStmt: q.evaluateStringAttributeValueLowerThanStmt, - evaluateStringAttributeValueNotEqualStmt: q.evaluateStringAttributeValueNotEqualStmt, - evaluateStringAttributeValueNotGlobStmt: q.evaluateStringAttributeValueNotGlobStmt, - evaluateStringAttributeValueNotInclusionStmt: q.evaluateStringAttributeValueNotInclusionStmt, - getLastBlockStmt: q.getLastBlockStmt, - getNumberOfEntitiesStmt: q.getNumberOfEntitiesStmt, - getNumericAttributeValueBitmapStmt: q.getNumericAttributeValueBitmapStmt, - getPayloadForEntityKeyStmt: q.getPayloadForEntityKeyStmt, - getStringAttributeValueBitmapStmt: q.getStringAttributeValueBitmapStmt, - retrievePayloadsStmt: q.retrievePayloadsStmt, - upsertLastBlockStmt: q.upsertLastBlockStmt, - upsertNumericAttributeValueBitmapStmt: q.upsertNumericAttributeValueBitmapStmt, - upsertPayloadStmt: q.upsertPayloadStmt, - upsertStringAttributeValueBitmapStmt: q.upsertStringAttributeValueBitmapStmt, + db: tx, + tx: tx, + closePayloadVersionStmt: q.closePayloadVersionStmt, + countDeltasSinceLastKeyframeNumericStmt: q.countDeltasSinceLastKeyframeNumericStmt, + countDeltasSinceLastKeyframeStringStmt: q.countDeltasSinceLastKeyframeStringStmt, + evaluateAllAtBlockStmt: q.evaluateAllAtBlockStmt, + evaluateAllCurrentStmt: q.evaluateAllCurrentStmt, + getAllNumericBitmapEntriesFromStmt: q.getAllNumericBitmapEntriesFromStmt, + getAllStringBitmapEntriesFromStmt: q.getAllStringBitmapEntriesFromStmt, + getCurrentPayloadForEntityKeyStmt: q.getCurrentPayloadForEntityKeyStmt, + getLastBlockStmt: q.getLastBlockStmt, + getLatestNumericKeyframeBlockStmt: q.getLatestNumericKeyframeBlockStmt, + getLatestStringKeyframeBlockStmt: q.getLatestStringKeyframeBlockStmt, + getMatchingNumericValuesEqualStmt: q.getMatchingNumericValuesEqualStmt, + getMatchingNumericValuesGreaterOrEqualThanStmt: q.getMatchingNumericValuesGreaterOrEqualThanStmt, + getMatchingNumericValuesGreaterThanStmt: q.getMatchingNumericValuesGreaterThanStmt, + getMatchingNumericValuesInclusionStmt: q.getMatchingNumericValuesInclusionStmt, + getMatchingNumericValuesLessOrEqualThanStmt: q.getMatchingNumericValuesLessOrEqualThanStmt, + getMatchingNumericValuesLessThanStmt: q.getMatchingNumericValuesLessThanStmt, + getMatchingNumericValuesNotEqualStmt: q.getMatchingNumericValuesNotEqualStmt, + getMatchingNumericValuesNotInclusionStmt: q.getMatchingNumericValuesNotInclusionStmt, + getMatchingStringValuesEqualStmt: q.getMatchingStringValuesEqualStmt, + getMatchingStringValuesGlobStmt: q.getMatchingStringValuesGlobStmt, + getMatchingStringValuesGreaterOrEqualThanStmt: q.getMatchingStringValuesGreaterOrEqualThanStmt, + getMatchingStringValuesGreaterThanStmt: q.getMatchingStringValuesGreaterThanStmt, + getMatchingStringValuesInclusionStmt: q.getMatchingStringValuesInclusionStmt, + getMatchingStringValuesLessOrEqualThanStmt: q.getMatchingStringValuesLessOrEqualThanStmt, + getMatchingStringValuesLessThanStmt: q.getMatchingStringValuesLessThanStmt, + getMatchingStringValuesNotEqualStmt: q.getMatchingStringValuesNotEqualStmt, + getMatchingStringValuesNotGlobStmt: q.getMatchingStringValuesNotGlobStmt, + getMatchingStringValuesNotInclusionStmt: q.getMatchingStringValuesNotInclusionStmt, + getNumberOfEntitiesStmt: q.getNumberOfEntitiesStmt, + getNumericBitmapEntriesInRangeStmt: q.getNumericBitmapEntriesInRangeStmt, + getNumericKeyframeBlockAtOrBeforeStmt: q.getNumericKeyframeBlockAtOrBeforeStmt, + getStringBitmapEntriesInRangeStmt: q.getStringBitmapEntriesInRangeStmt, + getStringKeyframeBlockAtOrBeforeStmt: q.getStringKeyframeBlockAtOrBeforeStmt, + insertNumericBitmapEntryStmt: q.insertNumericBitmapEntryStmt, + insertPayloadStmt: q.insertPayloadStmt, + insertStringBitmapEntryStmt: q.insertStringBitmapEntryStmt, + pruneNumericBitmapsBeforeStmt: q.pruneNumericBitmapsBeforeStmt, + prunePayloadsBeforeStmt: q.prunePayloadsBeforeStmt, + pruneStringBitmapsBeforeStmt: q.pruneStringBitmapsBeforeStmt, + reorgDeleteNewPayloadsStmt: q.reorgDeleteNewPayloadsStmt, + reorgDeleteNumericBitmapsStmt: q.reorgDeleteNumericBitmapsStmt, + reorgDeleteStringBitmapsStmt: q.reorgDeleteStringBitmapsStmt, + reorgReopenPayloadsStmt: q.reorgReopenPayloadsStmt, + retrievePayloadsStmt: q.retrievePayloadsStmt, + upsertLastBlockStmt: q.upsertLastBlockStmt, } } diff --git a/store/evals.sql.go b/store/evals.sql.go index 2d10e04..1de5eb3 100644 --- a/store/evals.sql.go +++ b/store/evals.sql.go @@ -10,24 +10,33 @@ import ( "strings" ) -const evaluateAll = `-- name: EvaluateAll :many -SELECT id FROM payloads -ORDER BY id DESC +const getMatchingNumericValuesEqual = `-- name: GetMatchingNumericValuesEqual :many + +SELECT DISTINCT value FROM numeric_attributes_values_bitmaps +WHERE name = ?1 AND value = ?2 + AND block <= ?3 ` -func (q *Queries) EvaluateAll(ctx context.Context) ([]uint64, error) { - rows, err := q.query(ctx, q.evaluateAllStmt, evaluateAll) +type GetMatchingNumericValuesEqualParams struct { + Name string + Value uint64 + TargetBlock uint64 +} + +// Numeric attribute value queries +func (q *Queries) GetMatchingNumericValuesEqual(ctx context.Context, arg GetMatchingNumericValuesEqualParams) ([]uint64, error) { + rows, err := q.query(ctx, q.getMatchingNumericValuesEqualStmt, getMatchingNumericValuesEqual, arg.Name, arg.Value, arg.TargetBlock) if err != nil { return nil, err } defer rows.Close() items := []uint64{} for rows.Next() { - var id uint64 - if err := rows.Scan(&id); err != nil { + var value uint64 + if err := rows.Scan(&value); err != nil { return nil, err } - items = append(items, id) + items = append(items, value) } if err := rows.Close(); err != nil { return nil, err @@ -38,46 +47,31 @@ func (q *Queries) EvaluateAll(ctx context.Context) ([]uint64, error) { return items, nil } -const evaluateNumericAttributeValueEqual = `-- name: EvaluateNumericAttributeValueEqual :one -SELECT bitmap FROM numeric_attributes_values_bitmaps -WHERE name = ?1 AND value = ?2 -` - -type EvaluateNumericAttributeValueEqualParams struct { - Name string - Value uint64 -} - -func (q *Queries) EvaluateNumericAttributeValueEqual(ctx context.Context, arg EvaluateNumericAttributeValueEqualParams) (*Bitmap, error) { - row := q.queryRow(ctx, q.evaluateNumericAttributeValueEqualStmt, evaluateNumericAttributeValueEqual, arg.Name, arg.Value) - var bitmap *Bitmap - err := row.Scan(&bitmap) - return bitmap, err -} - -const evaluateNumericAttributeValueGreaterOrEqualThan = `-- name: EvaluateNumericAttributeValueGreaterOrEqualThan :many -SELECT bitmap FROM numeric_attributes_values_bitmaps +const getMatchingNumericValuesGreaterOrEqualThan = `-- name: GetMatchingNumericValuesGreaterOrEqualThan :many +SELECT DISTINCT value FROM numeric_attributes_values_bitmaps WHERE name = ?1 AND value >= ?2 + AND block <= ?3 ` -type EvaluateNumericAttributeValueGreaterOrEqualThanParams struct { - Name string - Value uint64 +type GetMatchingNumericValuesGreaterOrEqualThanParams struct { + Name string + Value uint64 + TargetBlock uint64 } -func (q *Queries) EvaluateNumericAttributeValueGreaterOrEqualThan(ctx context.Context, arg EvaluateNumericAttributeValueGreaterOrEqualThanParams) ([]*Bitmap, error) { - rows, err := q.query(ctx, q.evaluateNumericAttributeValueGreaterOrEqualThanStmt, evaluateNumericAttributeValueGreaterOrEqualThan, arg.Name, arg.Value) +func (q *Queries) GetMatchingNumericValuesGreaterOrEqualThan(ctx context.Context, arg GetMatchingNumericValuesGreaterOrEqualThanParams) ([]uint64, error) { + rows, err := q.query(ctx, q.getMatchingNumericValuesGreaterOrEqualThanStmt, getMatchingNumericValuesGreaterOrEqualThan, arg.Name, arg.Value, arg.TargetBlock) if err != nil { return nil, err } defer rows.Close() - items := []*Bitmap{} + items := []uint64{} for rows.Next() { - var bitmap *Bitmap - if err := rows.Scan(&bitmap); err != nil { + var value uint64 + if err := rows.Scan(&value); err != nil { return nil, err } - items = append(items, bitmap) + items = append(items, value) } if err := rows.Close(); err != nil { return nil, err @@ -88,29 +82,31 @@ func (q *Queries) EvaluateNumericAttributeValueGreaterOrEqualThan(ctx context.Co return items, nil } -const evaluateNumericAttributeValueGreaterThan = `-- name: EvaluateNumericAttributeValueGreaterThan :many -SELECT bitmap FROM numeric_attributes_values_bitmaps +const getMatchingNumericValuesGreaterThan = `-- name: GetMatchingNumericValuesGreaterThan :many +SELECT DISTINCT value FROM numeric_attributes_values_bitmaps WHERE name = ?1 AND value > ?2 + AND block <= ?3 ` -type EvaluateNumericAttributeValueGreaterThanParams struct { - Name string - Value uint64 +type GetMatchingNumericValuesGreaterThanParams struct { + Name string + Value uint64 + TargetBlock uint64 } -func (q *Queries) EvaluateNumericAttributeValueGreaterThan(ctx context.Context, arg EvaluateNumericAttributeValueGreaterThanParams) ([]*Bitmap, error) { - rows, err := q.query(ctx, q.evaluateNumericAttributeValueGreaterThanStmt, evaluateNumericAttributeValueGreaterThan, arg.Name, arg.Value) +func (q *Queries) GetMatchingNumericValuesGreaterThan(ctx context.Context, arg GetMatchingNumericValuesGreaterThanParams) ([]uint64, error) { + rows, err := q.query(ctx, q.getMatchingNumericValuesGreaterThanStmt, getMatchingNumericValuesGreaterThan, arg.Name, arg.Value, arg.TargetBlock) if err != nil { return nil, err } defer rows.Close() - items := []*Bitmap{} + items := []uint64{} for rows.Next() { - var bitmap *Bitmap - if err := rows.Scan(&bitmap); err != nil { + var value uint64 + if err := rows.Scan(&value); err != nil { return nil, err } - items = append(items, bitmap) + items = append(items, value) } if err := rows.Close(); err != nil { return nil, err @@ -121,18 +117,20 @@ func (q *Queries) EvaluateNumericAttributeValueGreaterThan(ctx context.Context, return items, nil } -const evaluateNumericAttributeValueInclusion = `-- name: EvaluateNumericAttributeValueInclusion :many -SELECT bitmap FROM numeric_attributes_values_bitmaps +const getMatchingNumericValuesInclusion = `-- name: GetMatchingNumericValuesInclusion :many +SELECT DISTINCT value FROM numeric_attributes_values_bitmaps WHERE name = ?1 AND value IN (/*SLICE:values*/?) + AND block <= ?3 ` -type EvaluateNumericAttributeValueInclusionParams struct { - Name string - Values []uint64 +type GetMatchingNumericValuesInclusionParams struct { + Name string + Values []uint64 + TargetBlock uint64 } -func (q *Queries) EvaluateNumericAttributeValueInclusion(ctx context.Context, arg EvaluateNumericAttributeValueInclusionParams) ([]*Bitmap, error) { - query := evaluateNumericAttributeValueInclusion +func (q *Queries) GetMatchingNumericValuesInclusion(ctx context.Context, arg GetMatchingNumericValuesInclusionParams) ([]uint64, error) { + query := getMatchingNumericValuesInclusion var queryParams []interface{} queryParams = append(queryParams, arg.Name) if len(arg.Values) > 0 { @@ -143,18 +141,19 @@ func (q *Queries) EvaluateNumericAttributeValueInclusion(ctx context.Context, ar } else { query = strings.Replace(query, "/*SLICE:values*/?", "NULL", 1) } + queryParams = append(queryParams, arg.TargetBlock) rows, err := q.query(ctx, nil, query, queryParams...) if err != nil { return nil, err } defer rows.Close() - items := []*Bitmap{} + items := []uint64{} for rows.Next() { - var bitmap *Bitmap - if err := rows.Scan(&bitmap); err != nil { + var value uint64 + if err := rows.Scan(&value); err != nil { return nil, err } - items = append(items, bitmap) + items = append(items, value) } if err := rows.Close(); err != nil { return nil, err @@ -165,29 +164,31 @@ func (q *Queries) EvaluateNumericAttributeValueInclusion(ctx context.Context, ar return items, nil } -const evaluateNumericAttributeValueLessOrEqualThan = `-- name: EvaluateNumericAttributeValueLessOrEqualThan :many -SELECT bitmap FROM numeric_attributes_values_bitmaps +const getMatchingNumericValuesLessOrEqualThan = `-- name: GetMatchingNumericValuesLessOrEqualThan :many +SELECT DISTINCT value FROM numeric_attributes_values_bitmaps WHERE name = ?1 AND value <= ?2 + AND block <= ?3 ` -type EvaluateNumericAttributeValueLessOrEqualThanParams struct { - Name string - Value uint64 +type GetMatchingNumericValuesLessOrEqualThanParams struct { + Name string + Value uint64 + TargetBlock uint64 } -func (q *Queries) EvaluateNumericAttributeValueLessOrEqualThan(ctx context.Context, arg EvaluateNumericAttributeValueLessOrEqualThanParams) ([]*Bitmap, error) { - rows, err := q.query(ctx, q.evaluateNumericAttributeValueLessOrEqualThanStmt, evaluateNumericAttributeValueLessOrEqualThan, arg.Name, arg.Value) +func (q *Queries) GetMatchingNumericValuesLessOrEqualThan(ctx context.Context, arg GetMatchingNumericValuesLessOrEqualThanParams) ([]uint64, error) { + rows, err := q.query(ctx, q.getMatchingNumericValuesLessOrEqualThanStmt, getMatchingNumericValuesLessOrEqualThan, arg.Name, arg.Value, arg.TargetBlock) if err != nil { return nil, err } defer rows.Close() - items := []*Bitmap{} + items := []uint64{} for rows.Next() { - var bitmap *Bitmap - if err := rows.Scan(&bitmap); err != nil { + var value uint64 + if err := rows.Scan(&value); err != nil { return nil, err } - items = append(items, bitmap) + items = append(items, value) } if err := rows.Close(); err != nil { return nil, err @@ -198,29 +199,31 @@ func (q *Queries) EvaluateNumericAttributeValueLessOrEqualThan(ctx context.Conte return items, nil } -const evaluateNumericAttributeValueLowerThan = `-- name: EvaluateNumericAttributeValueLowerThan :many -SELECT bitmap FROM numeric_attributes_values_bitmaps +const getMatchingNumericValuesLessThan = `-- name: GetMatchingNumericValuesLessThan :many +SELECT DISTINCT value FROM numeric_attributes_values_bitmaps WHERE name = ?1 AND value < ?2 + AND block <= ?3 ` -type EvaluateNumericAttributeValueLowerThanParams struct { - Name string - Value uint64 +type GetMatchingNumericValuesLessThanParams struct { + Name string + Value uint64 + TargetBlock uint64 } -func (q *Queries) EvaluateNumericAttributeValueLowerThan(ctx context.Context, arg EvaluateNumericAttributeValueLowerThanParams) ([]*Bitmap, error) { - rows, err := q.query(ctx, q.evaluateNumericAttributeValueLowerThanStmt, evaluateNumericAttributeValueLowerThan, arg.Name, arg.Value) +func (q *Queries) GetMatchingNumericValuesLessThan(ctx context.Context, arg GetMatchingNumericValuesLessThanParams) ([]uint64, error) { + rows, err := q.query(ctx, q.getMatchingNumericValuesLessThanStmt, getMatchingNumericValuesLessThan, arg.Name, arg.Value, arg.TargetBlock) if err != nil { return nil, err } defer rows.Close() - items := []*Bitmap{} + items := []uint64{} for rows.Next() { - var bitmap *Bitmap - if err := rows.Scan(&bitmap); err != nil { + var value uint64 + if err := rows.Scan(&value); err != nil { return nil, err } - items = append(items, bitmap) + items = append(items, value) } if err := rows.Close(); err != nil { return nil, err @@ -231,29 +234,31 @@ func (q *Queries) EvaluateNumericAttributeValueLowerThan(ctx context.Context, ar return items, nil } -const evaluateNumericAttributeValueNotEqual = `-- name: EvaluateNumericAttributeValueNotEqual :many -SELECT bitmap FROM numeric_attributes_values_bitmaps +const getMatchingNumericValuesNotEqual = `-- name: GetMatchingNumericValuesNotEqual :many +SELECT DISTINCT value FROM numeric_attributes_values_bitmaps WHERE name = ?1 AND value != ?2 + AND block <= ?3 ` -type EvaluateNumericAttributeValueNotEqualParams struct { - Name string - Value uint64 +type GetMatchingNumericValuesNotEqualParams struct { + Name string + Value uint64 + TargetBlock uint64 } -func (q *Queries) EvaluateNumericAttributeValueNotEqual(ctx context.Context, arg EvaluateNumericAttributeValueNotEqualParams) ([]*Bitmap, error) { - rows, err := q.query(ctx, q.evaluateNumericAttributeValueNotEqualStmt, evaluateNumericAttributeValueNotEqual, arg.Name, arg.Value) +func (q *Queries) GetMatchingNumericValuesNotEqual(ctx context.Context, arg GetMatchingNumericValuesNotEqualParams) ([]uint64, error) { + rows, err := q.query(ctx, q.getMatchingNumericValuesNotEqualStmt, getMatchingNumericValuesNotEqual, arg.Name, arg.Value, arg.TargetBlock) if err != nil { return nil, err } defer rows.Close() - items := []*Bitmap{} + items := []uint64{} for rows.Next() { - var bitmap *Bitmap - if err := rows.Scan(&bitmap); err != nil { + var value uint64 + if err := rows.Scan(&value); err != nil { return nil, err } - items = append(items, bitmap) + items = append(items, value) } if err := rows.Close(); err != nil { return nil, err @@ -264,18 +269,20 @@ func (q *Queries) EvaluateNumericAttributeValueNotEqual(ctx context.Context, arg return items, nil } -const evaluateNumericAttributeValueNotInclusion = `-- name: EvaluateNumericAttributeValueNotInclusion :many -SELECT bitmap FROM numeric_attributes_values_bitmaps +const getMatchingNumericValuesNotInclusion = `-- name: GetMatchingNumericValuesNotInclusion :many +SELECT DISTINCT value FROM numeric_attributes_values_bitmaps WHERE name = ?1 AND value NOT IN (/*SLICE:values*/?) + AND block <= ?3 ` -type EvaluateNumericAttributeValueNotInclusionParams struct { - Name string - Values []uint64 +type GetMatchingNumericValuesNotInclusionParams struct { + Name string + Values []uint64 + TargetBlock uint64 } -func (q *Queries) EvaluateNumericAttributeValueNotInclusion(ctx context.Context, arg EvaluateNumericAttributeValueNotInclusionParams) ([]*Bitmap, error) { - query := evaluateNumericAttributeValueNotInclusion +func (q *Queries) GetMatchingNumericValuesNotInclusion(ctx context.Context, arg GetMatchingNumericValuesNotInclusionParams) ([]uint64, error) { + query := getMatchingNumericValuesNotInclusion var queryParams []interface{} queryParams = append(queryParams, arg.Name) if len(arg.Values) > 0 { @@ -286,18 +293,19 @@ func (q *Queries) EvaluateNumericAttributeValueNotInclusion(ctx context.Context, } else { query = strings.Replace(query, "/*SLICE:values*/?", "NULL", 1) } + queryParams = append(queryParams, arg.TargetBlock) rows, err := q.query(ctx, nil, query, queryParams...) if err != nil { return nil, err } defer rows.Close() - items := []*Bitmap{} + items := []uint64{} for rows.Next() { - var bitmap *Bitmap - if err := rows.Scan(&bitmap); err != nil { + var value uint64 + if err := rows.Scan(&value); err != nil { return nil, err } - items = append(items, bitmap) + items = append(items, value) } if err := rows.Close(); err != nil { return nil, err @@ -308,46 +316,72 @@ func (q *Queries) EvaluateNumericAttributeValueNotInclusion(ctx context.Context, return items, nil } -const evaluateStringAttributeValueEqual = `-- name: EvaluateStringAttributeValueEqual :one -SELECT bitmap FROM string_attributes_values_bitmaps +const getMatchingStringValuesEqual = `-- name: GetMatchingStringValuesEqual :many + + +SELECT DISTINCT value FROM string_attributes_values_bitmaps WHERE name = ?1 AND value = ?2 + AND block <= ?3 ` -type EvaluateStringAttributeValueEqualParams struct { - Name string - Value string +type GetMatchingStringValuesEqualParams struct { + Name string + Value string + TargetBlock uint64 } -func (q *Queries) EvaluateStringAttributeValueEqual(ctx context.Context, arg EvaluateStringAttributeValueEqualParams) (*Bitmap, error) { - row := q.queryRow(ctx, q.evaluateStringAttributeValueEqualStmt, evaluateStringAttributeValueEqual, arg.Name, arg.Value) - var bitmap *Bitmap - err := row.Scan(&bitmap) - return bitmap, err +// Value-listing queries for query evaluation. +// Each returns distinct values matching a condition, filtered by block. +// The Go layer then reconstructs bitmaps for each matching value via XOR chains. +// String attribute value queries +func (q *Queries) GetMatchingStringValuesEqual(ctx context.Context, arg GetMatchingStringValuesEqualParams) ([]string, error) { + rows, err := q.query(ctx, q.getMatchingStringValuesEqualStmt, getMatchingStringValuesEqual, arg.Name, arg.Value, arg.TargetBlock) + if err != nil { + return nil, err + } + defer rows.Close() + items := []string{} + for rows.Next() { + var value string + if err := rows.Scan(&value); err != nil { + return nil, err + } + items = append(items, value) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil } -const evaluateStringAttributeValueGlob = `-- name: EvaluateStringAttributeValueGlob :many -SELECT bitmap FROM string_attributes_values_bitmaps +const getMatchingStringValuesGlob = `-- name: GetMatchingStringValuesGlob :many +SELECT DISTINCT value FROM string_attributes_values_bitmaps WHERE name = ?1 AND value GLOB ?2 + AND block <= ?3 ` -type EvaluateStringAttributeValueGlobParams struct { - Name string - Value string +type GetMatchingStringValuesGlobParams struct { + Name string + Value string + TargetBlock uint64 } -func (q *Queries) EvaluateStringAttributeValueGlob(ctx context.Context, arg EvaluateStringAttributeValueGlobParams) ([]*Bitmap, error) { - rows, err := q.query(ctx, q.evaluateStringAttributeValueGlobStmt, evaluateStringAttributeValueGlob, arg.Name, arg.Value) +func (q *Queries) GetMatchingStringValuesGlob(ctx context.Context, arg GetMatchingStringValuesGlobParams) ([]string, error) { + rows, err := q.query(ctx, q.getMatchingStringValuesGlobStmt, getMatchingStringValuesGlob, arg.Name, arg.Value, arg.TargetBlock) if err != nil { return nil, err } defer rows.Close() - items := []*Bitmap{} + items := []string{} for rows.Next() { - var bitmap *Bitmap - if err := rows.Scan(&bitmap); err != nil { + var value string + if err := rows.Scan(&value); err != nil { return nil, err } - items = append(items, bitmap) + items = append(items, value) } if err := rows.Close(); err != nil { return nil, err @@ -358,29 +392,31 @@ func (q *Queries) EvaluateStringAttributeValueGlob(ctx context.Context, arg Eval return items, nil } -const evaluateStringAttributeValueGreaterOrEqualThan = `-- name: EvaluateStringAttributeValueGreaterOrEqualThan :many -SELECT bitmap FROM string_attributes_values_bitmaps +const getMatchingStringValuesGreaterOrEqualThan = `-- name: GetMatchingStringValuesGreaterOrEqualThan :many +SELECT DISTINCT value FROM string_attributes_values_bitmaps WHERE name = ?1 AND value >= ?2 + AND block <= ?3 ` -type EvaluateStringAttributeValueGreaterOrEqualThanParams struct { - Name string - Value string +type GetMatchingStringValuesGreaterOrEqualThanParams struct { + Name string + Value string + TargetBlock uint64 } -func (q *Queries) EvaluateStringAttributeValueGreaterOrEqualThan(ctx context.Context, arg EvaluateStringAttributeValueGreaterOrEqualThanParams) ([]*Bitmap, error) { - rows, err := q.query(ctx, q.evaluateStringAttributeValueGreaterOrEqualThanStmt, evaluateStringAttributeValueGreaterOrEqualThan, arg.Name, arg.Value) +func (q *Queries) GetMatchingStringValuesGreaterOrEqualThan(ctx context.Context, arg GetMatchingStringValuesGreaterOrEqualThanParams) ([]string, error) { + rows, err := q.query(ctx, q.getMatchingStringValuesGreaterOrEqualThanStmt, getMatchingStringValuesGreaterOrEqualThan, arg.Name, arg.Value, arg.TargetBlock) if err != nil { return nil, err } defer rows.Close() - items := []*Bitmap{} + items := []string{} for rows.Next() { - var bitmap *Bitmap - if err := rows.Scan(&bitmap); err != nil { + var value string + if err := rows.Scan(&value); err != nil { return nil, err } - items = append(items, bitmap) + items = append(items, value) } if err := rows.Close(); err != nil { return nil, err @@ -391,29 +427,31 @@ func (q *Queries) EvaluateStringAttributeValueGreaterOrEqualThan(ctx context.Con return items, nil } -const evaluateStringAttributeValueGreaterThan = `-- name: EvaluateStringAttributeValueGreaterThan :many -SELECT bitmap FROM string_attributes_values_bitmaps +const getMatchingStringValuesGreaterThan = `-- name: GetMatchingStringValuesGreaterThan :many +SELECT DISTINCT value FROM string_attributes_values_bitmaps WHERE name = ?1 AND value > ?2 + AND block <= ?3 ` -type EvaluateStringAttributeValueGreaterThanParams struct { - Name string - Value string +type GetMatchingStringValuesGreaterThanParams struct { + Name string + Value string + TargetBlock uint64 } -func (q *Queries) EvaluateStringAttributeValueGreaterThan(ctx context.Context, arg EvaluateStringAttributeValueGreaterThanParams) ([]*Bitmap, error) { - rows, err := q.query(ctx, q.evaluateStringAttributeValueGreaterThanStmt, evaluateStringAttributeValueGreaterThan, arg.Name, arg.Value) +func (q *Queries) GetMatchingStringValuesGreaterThan(ctx context.Context, arg GetMatchingStringValuesGreaterThanParams) ([]string, error) { + rows, err := q.query(ctx, q.getMatchingStringValuesGreaterThanStmt, getMatchingStringValuesGreaterThan, arg.Name, arg.Value, arg.TargetBlock) if err != nil { return nil, err } defer rows.Close() - items := []*Bitmap{} + items := []string{} for rows.Next() { - var bitmap *Bitmap - if err := rows.Scan(&bitmap); err != nil { + var value string + if err := rows.Scan(&value); err != nil { return nil, err } - items = append(items, bitmap) + items = append(items, value) } if err := rows.Close(); err != nil { return nil, err @@ -424,18 +462,20 @@ func (q *Queries) EvaluateStringAttributeValueGreaterThan(ctx context.Context, a return items, nil } -const evaluateStringAttributeValueInclusion = `-- name: EvaluateStringAttributeValueInclusion :many -SELECT bitmap FROM string_attributes_values_bitmaps +const getMatchingStringValuesInclusion = `-- name: GetMatchingStringValuesInclusion :many +SELECT DISTINCT value FROM string_attributes_values_bitmaps WHERE name = ?1 AND value IN (/*SLICE:values*/?) + AND block <= ?3 ` -type EvaluateStringAttributeValueInclusionParams struct { - Name string - Values []string +type GetMatchingStringValuesInclusionParams struct { + Name string + Values []string + TargetBlock uint64 } -func (q *Queries) EvaluateStringAttributeValueInclusion(ctx context.Context, arg EvaluateStringAttributeValueInclusionParams) ([]*Bitmap, error) { - query := evaluateStringAttributeValueInclusion +func (q *Queries) GetMatchingStringValuesInclusion(ctx context.Context, arg GetMatchingStringValuesInclusionParams) ([]string, error) { + query := getMatchingStringValuesInclusion var queryParams []interface{} queryParams = append(queryParams, arg.Name) if len(arg.Values) > 0 { @@ -446,18 +486,19 @@ func (q *Queries) EvaluateStringAttributeValueInclusion(ctx context.Context, arg } else { query = strings.Replace(query, "/*SLICE:values*/?", "NULL", 1) } + queryParams = append(queryParams, arg.TargetBlock) rows, err := q.query(ctx, nil, query, queryParams...) if err != nil { return nil, err } defer rows.Close() - items := []*Bitmap{} + items := []string{} for rows.Next() { - var bitmap *Bitmap - if err := rows.Scan(&bitmap); err != nil { + var value string + if err := rows.Scan(&value); err != nil { return nil, err } - items = append(items, bitmap) + items = append(items, value) } if err := rows.Close(); err != nil { return nil, err @@ -468,29 +509,31 @@ func (q *Queries) EvaluateStringAttributeValueInclusion(ctx context.Context, arg return items, nil } -const evaluateStringAttributeValueLessOrEqualThan = `-- name: EvaluateStringAttributeValueLessOrEqualThan :many -SELECT bitmap FROM string_attributes_values_bitmaps +const getMatchingStringValuesLessOrEqualThan = `-- name: GetMatchingStringValuesLessOrEqualThan :many +SELECT DISTINCT value FROM string_attributes_values_bitmaps WHERE name = ?1 AND value <= ?2 + AND block <= ?3 ` -type EvaluateStringAttributeValueLessOrEqualThanParams struct { - Name string - Value string +type GetMatchingStringValuesLessOrEqualThanParams struct { + Name string + Value string + TargetBlock uint64 } -func (q *Queries) EvaluateStringAttributeValueLessOrEqualThan(ctx context.Context, arg EvaluateStringAttributeValueLessOrEqualThanParams) ([]*Bitmap, error) { - rows, err := q.query(ctx, q.evaluateStringAttributeValueLessOrEqualThanStmt, evaluateStringAttributeValueLessOrEqualThan, arg.Name, arg.Value) +func (q *Queries) GetMatchingStringValuesLessOrEqualThan(ctx context.Context, arg GetMatchingStringValuesLessOrEqualThanParams) ([]string, error) { + rows, err := q.query(ctx, q.getMatchingStringValuesLessOrEqualThanStmt, getMatchingStringValuesLessOrEqualThan, arg.Name, arg.Value, arg.TargetBlock) if err != nil { return nil, err } defer rows.Close() - items := []*Bitmap{} + items := []string{} for rows.Next() { - var bitmap *Bitmap - if err := rows.Scan(&bitmap); err != nil { + var value string + if err := rows.Scan(&value); err != nil { return nil, err } - items = append(items, bitmap) + items = append(items, value) } if err := rows.Close(); err != nil { return nil, err @@ -501,29 +544,31 @@ func (q *Queries) EvaluateStringAttributeValueLessOrEqualThan(ctx context.Contex return items, nil } -const evaluateStringAttributeValueLowerThan = `-- name: EvaluateStringAttributeValueLowerThan :many -SELECT bitmap FROM string_attributes_values_bitmaps +const getMatchingStringValuesLessThan = `-- name: GetMatchingStringValuesLessThan :many +SELECT DISTINCT value FROM string_attributes_values_bitmaps WHERE name = ?1 AND value < ?2 + AND block <= ?3 ` -type EvaluateStringAttributeValueLowerThanParams struct { - Name string - Value string +type GetMatchingStringValuesLessThanParams struct { + Name string + Value string + TargetBlock uint64 } -func (q *Queries) EvaluateStringAttributeValueLowerThan(ctx context.Context, arg EvaluateStringAttributeValueLowerThanParams) ([]*Bitmap, error) { - rows, err := q.query(ctx, q.evaluateStringAttributeValueLowerThanStmt, evaluateStringAttributeValueLowerThan, arg.Name, arg.Value) +func (q *Queries) GetMatchingStringValuesLessThan(ctx context.Context, arg GetMatchingStringValuesLessThanParams) ([]string, error) { + rows, err := q.query(ctx, q.getMatchingStringValuesLessThanStmt, getMatchingStringValuesLessThan, arg.Name, arg.Value, arg.TargetBlock) if err != nil { return nil, err } defer rows.Close() - items := []*Bitmap{} + items := []string{} for rows.Next() { - var bitmap *Bitmap - if err := rows.Scan(&bitmap); err != nil { + var value string + if err := rows.Scan(&value); err != nil { return nil, err } - items = append(items, bitmap) + items = append(items, value) } if err := rows.Close(); err != nil { return nil, err @@ -534,29 +579,31 @@ func (q *Queries) EvaluateStringAttributeValueLowerThan(ctx context.Context, arg return items, nil } -const evaluateStringAttributeValueNotEqual = `-- name: EvaluateStringAttributeValueNotEqual :many -SELECT bitmap FROM string_attributes_values_bitmaps +const getMatchingStringValuesNotEqual = `-- name: GetMatchingStringValuesNotEqual :many +SELECT DISTINCT value FROM string_attributes_values_bitmaps WHERE name = ?1 AND value != ?2 + AND block <= ?3 ` -type EvaluateStringAttributeValueNotEqualParams struct { - Name string - Value string +type GetMatchingStringValuesNotEqualParams struct { + Name string + Value string + TargetBlock uint64 } -func (q *Queries) EvaluateStringAttributeValueNotEqual(ctx context.Context, arg EvaluateStringAttributeValueNotEqualParams) ([]*Bitmap, error) { - rows, err := q.query(ctx, q.evaluateStringAttributeValueNotEqualStmt, evaluateStringAttributeValueNotEqual, arg.Name, arg.Value) +func (q *Queries) GetMatchingStringValuesNotEqual(ctx context.Context, arg GetMatchingStringValuesNotEqualParams) ([]string, error) { + rows, err := q.query(ctx, q.getMatchingStringValuesNotEqualStmt, getMatchingStringValuesNotEqual, arg.Name, arg.Value, arg.TargetBlock) if err != nil { return nil, err } defer rows.Close() - items := []*Bitmap{} + items := []string{} for rows.Next() { - var bitmap *Bitmap - if err := rows.Scan(&bitmap); err != nil { + var value string + if err := rows.Scan(&value); err != nil { return nil, err } - items = append(items, bitmap) + items = append(items, value) } if err := rows.Close(); err != nil { return nil, err @@ -567,29 +614,31 @@ func (q *Queries) EvaluateStringAttributeValueNotEqual(ctx context.Context, arg return items, nil } -const evaluateStringAttributeValueNotGlob = `-- name: EvaluateStringAttributeValueNotGlob :many -SELECT bitmap FROM string_attributes_values_bitmaps +const getMatchingStringValuesNotGlob = `-- name: GetMatchingStringValuesNotGlob :many +SELECT DISTINCT value FROM string_attributes_values_bitmaps WHERE name = ?1 AND value NOT GLOB ?2 + AND block <= ?3 ` -type EvaluateStringAttributeValueNotGlobParams struct { - Name string - Value string +type GetMatchingStringValuesNotGlobParams struct { + Name string + Value string + TargetBlock uint64 } -func (q *Queries) EvaluateStringAttributeValueNotGlob(ctx context.Context, arg EvaluateStringAttributeValueNotGlobParams) ([]*Bitmap, error) { - rows, err := q.query(ctx, q.evaluateStringAttributeValueNotGlobStmt, evaluateStringAttributeValueNotGlob, arg.Name, arg.Value) +func (q *Queries) GetMatchingStringValuesNotGlob(ctx context.Context, arg GetMatchingStringValuesNotGlobParams) ([]string, error) { + rows, err := q.query(ctx, q.getMatchingStringValuesNotGlobStmt, getMatchingStringValuesNotGlob, arg.Name, arg.Value, arg.TargetBlock) if err != nil { return nil, err } defer rows.Close() - items := []*Bitmap{} + items := []string{} for rows.Next() { - var bitmap *Bitmap - if err := rows.Scan(&bitmap); err != nil { + var value string + if err := rows.Scan(&value); err != nil { return nil, err } - items = append(items, bitmap) + items = append(items, value) } if err := rows.Close(); err != nil { return nil, err @@ -600,18 +649,20 @@ func (q *Queries) EvaluateStringAttributeValueNotGlob(ctx context.Context, arg E return items, nil } -const evaluateStringAttributeValueNotInclusion = `-- name: EvaluateStringAttributeValueNotInclusion :many -SELECT bitmap FROM string_attributes_values_bitmaps +const getMatchingStringValuesNotInclusion = `-- name: GetMatchingStringValuesNotInclusion :many +SELECT DISTINCT value FROM string_attributes_values_bitmaps WHERE name = ?1 AND value NOT IN (/*SLICE:values*/?) + AND block <= ?3 ` -type EvaluateStringAttributeValueNotInclusionParams struct { - Name string - Values []string +type GetMatchingStringValuesNotInclusionParams struct { + Name string + Values []string + TargetBlock uint64 } -func (q *Queries) EvaluateStringAttributeValueNotInclusion(ctx context.Context, arg EvaluateStringAttributeValueNotInclusionParams) ([]*Bitmap, error) { - query := evaluateStringAttributeValueNotInclusion +func (q *Queries) GetMatchingStringValuesNotInclusion(ctx context.Context, arg GetMatchingStringValuesNotInclusionParams) ([]string, error) { + query := getMatchingStringValuesNotInclusion var queryParams []interface{} queryParams = append(queryParams, arg.Name) if len(arg.Values) > 0 { @@ -622,18 +673,19 @@ func (q *Queries) EvaluateStringAttributeValueNotInclusion(ctx context.Context, } else { query = strings.Replace(query, "/*SLICE:values*/?", "NULL", 1) } + queryParams = append(queryParams, arg.TargetBlock) rows, err := q.query(ctx, nil, query, queryParams...) if err != nil { return nil, err } defer rows.Close() - items := []*Bitmap{} + items := []string{} for rows.Next() { - var bitmap *Bitmap - if err := rows.Scan(&bitmap); err != nil { + var value string + if err := rows.Scan(&value); err != nil { return nil, err } - items = append(items, bitmap) + items = append(items, value) } if err := rows.Close(); err != nil { return nil, err diff --git a/store/models.go b/store/models.go index c6b351f..dc6462c 100644 --- a/store/models.go +++ b/store/models.go @@ -10,9 +10,11 @@ type LastBlock struct { } type NumericAttributesValuesBitmap struct { - Name string - Value uint64 - Bitmap *Bitmap + Name string + Value uint64 + Block uint64 + IsFullBitmap bool + Bitmap *Bitmap } type Payload struct { @@ -22,10 +24,14 @@ type Payload struct { ContentType string StringAttributes *StringAttributes NumericAttributes *NumericAttributes + FromBlock uint64 + ToBlock *uint64 } type StringAttributesValuesBitmap struct { - Name string - Value string - Bitmap *Bitmap + Name string + Value string + Block uint64 + IsFullBitmap bool + Bitmap *Bitmap } diff --git a/store/querier.go b/store/querier.go index a8e27ee..56db38a 100644 --- a/store/querier.go +++ b/store/querier.go @@ -9,38 +9,59 @@ import ( ) type Querier interface { - DeleteNumericAttributeValueBitmap(ctx context.Context, arg DeleteNumericAttributeValueBitmapParams) error - DeletePayloadForEntityKey(ctx context.Context, entityKey []byte) error - DeleteStringAttributeValueBitmap(ctx context.Context, arg DeleteStringAttributeValueBitmapParams) error - EvaluateAll(ctx context.Context) ([]uint64, error) - EvaluateNumericAttributeValueEqual(ctx context.Context, arg EvaluateNumericAttributeValueEqualParams) (*Bitmap, error) - EvaluateNumericAttributeValueGreaterOrEqualThan(ctx context.Context, arg EvaluateNumericAttributeValueGreaterOrEqualThanParams) ([]*Bitmap, error) - EvaluateNumericAttributeValueGreaterThan(ctx context.Context, arg EvaluateNumericAttributeValueGreaterThanParams) ([]*Bitmap, error) - EvaluateNumericAttributeValueInclusion(ctx context.Context, arg EvaluateNumericAttributeValueInclusionParams) ([]*Bitmap, error) - EvaluateNumericAttributeValueLessOrEqualThan(ctx context.Context, arg EvaluateNumericAttributeValueLessOrEqualThanParams) ([]*Bitmap, error) - EvaluateNumericAttributeValueLowerThan(ctx context.Context, arg EvaluateNumericAttributeValueLowerThanParams) ([]*Bitmap, error) - EvaluateNumericAttributeValueNotEqual(ctx context.Context, arg EvaluateNumericAttributeValueNotEqualParams) ([]*Bitmap, error) - EvaluateNumericAttributeValueNotInclusion(ctx context.Context, arg EvaluateNumericAttributeValueNotInclusionParams) ([]*Bitmap, error) - EvaluateStringAttributeValueEqual(ctx context.Context, arg EvaluateStringAttributeValueEqualParams) (*Bitmap, error) - EvaluateStringAttributeValueGlob(ctx context.Context, arg EvaluateStringAttributeValueGlobParams) ([]*Bitmap, error) - EvaluateStringAttributeValueGreaterOrEqualThan(ctx context.Context, arg EvaluateStringAttributeValueGreaterOrEqualThanParams) ([]*Bitmap, error) - EvaluateStringAttributeValueGreaterThan(ctx context.Context, arg EvaluateStringAttributeValueGreaterThanParams) ([]*Bitmap, error) - EvaluateStringAttributeValueInclusion(ctx context.Context, arg EvaluateStringAttributeValueInclusionParams) ([]*Bitmap, error) - EvaluateStringAttributeValueLessOrEqualThan(ctx context.Context, arg EvaluateStringAttributeValueLessOrEqualThanParams) ([]*Bitmap, error) - EvaluateStringAttributeValueLowerThan(ctx context.Context, arg EvaluateStringAttributeValueLowerThanParams) ([]*Bitmap, error) - EvaluateStringAttributeValueNotEqual(ctx context.Context, arg EvaluateStringAttributeValueNotEqualParams) ([]*Bitmap, error) - EvaluateStringAttributeValueNotGlob(ctx context.Context, arg EvaluateStringAttributeValueNotGlobParams) ([]*Bitmap, error) - EvaluateStringAttributeValueNotInclusion(ctx context.Context, arg EvaluateStringAttributeValueNotInclusionParams) ([]*Bitmap, error) + ClosePayloadVersion(ctx context.Context, arg ClosePayloadVersionParams) error + CountDeltasSinceLastKeyframeNumeric(ctx context.Context, arg CountDeltasSinceLastKeyframeNumericParams) (int64, error) + CountDeltasSinceLastKeyframeString(ctx context.Context, arg CountDeltasSinceLastKeyframeStringParams) (int64, error) + EvaluateAllAtBlock(ctx context.Context, block uint64) ([]uint64, error) + EvaluateAllCurrent(ctx context.Context) ([]uint64, error) + GetAllNumericBitmapEntriesFrom(ctx context.Context, arg GetAllNumericBitmapEntriesFromParams) ([]GetAllNumericBitmapEntriesFromRow, error) + GetAllStringBitmapEntriesFrom(ctx context.Context, arg GetAllStringBitmapEntriesFromParams) ([]GetAllStringBitmapEntriesFromRow, error) + GetCurrentPayloadForEntityKey(ctx context.Context, entityKey []byte) (GetCurrentPayloadForEntityKeyRow, error) GetLastBlock(ctx context.Context) (uint64, error) + GetLatestNumericKeyframeBlock(ctx context.Context, arg GetLatestNumericKeyframeBlockParams) (interface{}, error) + GetLatestStringKeyframeBlock(ctx context.Context, arg GetLatestStringKeyframeBlockParams) (interface{}, error) + // Numeric attribute value queries + GetMatchingNumericValuesEqual(ctx context.Context, arg GetMatchingNumericValuesEqualParams) ([]uint64, error) + GetMatchingNumericValuesGreaterOrEqualThan(ctx context.Context, arg GetMatchingNumericValuesGreaterOrEqualThanParams) ([]uint64, error) + GetMatchingNumericValuesGreaterThan(ctx context.Context, arg GetMatchingNumericValuesGreaterThanParams) ([]uint64, error) + GetMatchingNumericValuesInclusion(ctx context.Context, arg GetMatchingNumericValuesInclusionParams) ([]uint64, error) + GetMatchingNumericValuesLessOrEqualThan(ctx context.Context, arg GetMatchingNumericValuesLessOrEqualThanParams) ([]uint64, error) + GetMatchingNumericValuesLessThan(ctx context.Context, arg GetMatchingNumericValuesLessThanParams) ([]uint64, error) + GetMatchingNumericValuesNotEqual(ctx context.Context, arg GetMatchingNumericValuesNotEqualParams) ([]uint64, error) + GetMatchingNumericValuesNotInclusion(ctx context.Context, arg GetMatchingNumericValuesNotInclusionParams) ([]uint64, error) + // Value-listing queries for query evaluation. + // Each returns distinct values matching a condition, filtered by block. + // The Go layer then reconstructs bitmaps for each matching value via XOR chains. + // String attribute value queries + GetMatchingStringValuesEqual(ctx context.Context, arg GetMatchingStringValuesEqualParams) ([]string, error) + GetMatchingStringValuesGlob(ctx context.Context, arg GetMatchingStringValuesGlobParams) ([]string, error) + GetMatchingStringValuesGreaterOrEqualThan(ctx context.Context, arg GetMatchingStringValuesGreaterOrEqualThanParams) ([]string, error) + GetMatchingStringValuesGreaterThan(ctx context.Context, arg GetMatchingStringValuesGreaterThanParams) ([]string, error) + GetMatchingStringValuesInclusion(ctx context.Context, arg GetMatchingStringValuesInclusionParams) ([]string, error) + GetMatchingStringValuesLessOrEqualThan(ctx context.Context, arg GetMatchingStringValuesLessOrEqualThanParams) ([]string, error) + GetMatchingStringValuesLessThan(ctx context.Context, arg GetMatchingStringValuesLessThanParams) ([]string, error) + GetMatchingStringValuesNotEqual(ctx context.Context, arg GetMatchingStringValuesNotEqualParams) ([]string, error) + GetMatchingStringValuesNotGlob(ctx context.Context, arg GetMatchingStringValuesNotGlobParams) ([]string, error) + GetMatchingStringValuesNotInclusion(ctx context.Context, arg GetMatchingStringValuesNotInclusionParams) ([]string, error) GetNumberOfEntities(ctx context.Context) (int64, error) - GetNumericAttributeValueBitmap(ctx context.Context, arg GetNumericAttributeValueBitmapParams) (*Bitmap, error) - GetPayloadForEntityKey(ctx context.Context, entityKey []byte) (GetPayloadForEntityKeyRow, error) - GetStringAttributeValueBitmap(ctx context.Context, arg GetStringAttributeValueBitmapParams) (*Bitmap, error) + GetNumericBitmapEntriesInRange(ctx context.Context, arg GetNumericBitmapEntriesInRangeParams) ([]GetNumericBitmapEntriesInRangeRow, error) + GetNumericKeyframeBlockAtOrBefore(ctx context.Context, arg GetNumericKeyframeBlockAtOrBeforeParams) (interface{}, error) + GetStringBitmapEntriesInRange(ctx context.Context, arg GetStringBitmapEntriesInRangeParams) ([]GetStringBitmapEntriesInRangeRow, error) + GetStringKeyframeBlockAtOrBefore(ctx context.Context, arg GetStringKeyframeBlockAtOrBeforeParams) (interface{}, error) + InsertNumericBitmapEntry(ctx context.Context, arg InsertNumericBitmapEntryParams) error + InsertPayload(ctx context.Context, arg InsertPayloadParams) (uint64, error) + InsertStringBitmapEntry(ctx context.Context, arg InsertStringBitmapEntryParams) error + PruneNumericBitmapsBefore(ctx context.Context, pruneBlock uint64) error + // Pruning queries + PrunePayloadsBefore(ctx context.Context, block *uint64) error + PruneStringBitmapsBefore(ctx context.Context, pruneBlock uint64) error + ReorgDeleteNewPayloads(ctx context.Context, block uint64) error + ReorgDeleteNumericBitmaps(ctx context.Context, block uint64) error + ReorgDeleteStringBitmaps(ctx context.Context, block uint64) error + // Reorg queries + ReorgReopenPayloads(ctx context.Context, block uint64) error RetrievePayloads(ctx context.Context, ids []uint64) ([]RetrievePayloadsRow, error) UpsertLastBlock(ctx context.Context, block uint64) error - UpsertNumericAttributeValueBitmap(ctx context.Context, arg UpsertNumericAttributeValueBitmapParams) error - UpsertPayload(ctx context.Context, arg UpsertPayloadParams) (uint64, error) - UpsertStringAttributeValueBitmap(ctx context.Context, arg UpsertStringAttributeValueBitmapParams) error } var _ Querier = (*Queries)(nil) diff --git a/store/queries.sql.go b/store/queries.sql.go index 0788eb6..e6b3b44 100644 --- a/store/queries.sql.go +++ b/store/queries.sql.go @@ -9,92 +9,168 @@ import ( "context" ) -const deleteNumericAttributeValueBitmap = `-- name: DeleteNumericAttributeValueBitmap :exec -DELETE FROM numeric_attributes_values_bitmaps -WHERE name = ? AND value = ? +const closePayloadVersion = `-- name: ClosePayloadVersion :exec +UPDATE payloads SET to_block = ?1 +WHERE entity_key = ?2 AND to_block IS NULL ` -type DeleteNumericAttributeValueBitmapParams struct { - Name string - Value uint64 +type ClosePayloadVersionParams struct { + Block *uint64 + EntityKey []byte } -func (q *Queries) DeleteNumericAttributeValueBitmap(ctx context.Context, arg DeleteNumericAttributeValueBitmapParams) error { - _, err := q.exec(ctx, q.deleteNumericAttributeValueBitmapStmt, deleteNumericAttributeValueBitmap, arg.Name, arg.Value) +func (q *Queries) ClosePayloadVersion(ctx context.Context, arg ClosePayloadVersionParams) error { + _, err := q.exec(ctx, q.closePayloadVersionStmt, closePayloadVersion, arg.Block, arg.EntityKey) return err } -const deletePayloadForEntityKey = `-- name: DeletePayloadForEntityKey :exec -DELETE FROM payloads -WHERE entity_key = ? +const countDeltasSinceLastKeyframeNumeric = `-- name: CountDeltasSinceLastKeyframeNumeric :one +SELECT COUNT(*) FROM numeric_attributes_values_bitmaps AS outer_n +WHERE outer_n.name = ?1 AND outer_n.value = ?2 AND outer_n.is_full_bitmap = 0 + AND outer_n.block > ( + SELECT COALESCE(MAX(inner_n.block), -1) + FROM numeric_attributes_values_bitmaps AS inner_n + WHERE inner_n.name = ?1 AND inner_n.value = ?2 AND inner_n.is_full_bitmap = 1 + ) ` -func (q *Queries) DeletePayloadForEntityKey(ctx context.Context, entityKey []byte) error { - _, err := q.exec(ctx, q.deletePayloadForEntityKeyStmt, deletePayloadForEntityKey, entityKey) - return err +type CountDeltasSinceLastKeyframeNumericParams struct { + Name string + Value uint64 } -const deleteStringAttributeValueBitmap = `-- name: DeleteStringAttributeValueBitmap :exec -DELETE FROM string_attributes_values_bitmaps -WHERE name = ? AND value = ? +func (q *Queries) CountDeltasSinceLastKeyframeNumeric(ctx context.Context, arg CountDeltasSinceLastKeyframeNumericParams) (int64, error) { + row := q.queryRow(ctx, q.countDeltasSinceLastKeyframeNumericStmt, countDeltasSinceLastKeyframeNumeric, arg.Name, arg.Value) + var count int64 + err := row.Scan(&count) + return count, err +} + +const countDeltasSinceLastKeyframeString = `-- name: CountDeltasSinceLastKeyframeString :one +SELECT COUNT(*) FROM string_attributes_values_bitmaps AS outer_s +WHERE outer_s.name = ?1 AND outer_s.value = ?2 AND outer_s.is_full_bitmap = 0 + AND outer_s.block > ( + SELECT COALESCE(MAX(inner_s.block), -1) + FROM string_attributes_values_bitmaps AS inner_s + WHERE inner_s.name = ?1 AND inner_s.value = ?2 AND inner_s.is_full_bitmap = 1 + ) ` -type DeleteStringAttributeValueBitmapParams struct { +type CountDeltasSinceLastKeyframeStringParams struct { Name string Value string } -func (q *Queries) DeleteStringAttributeValueBitmap(ctx context.Context, arg DeleteStringAttributeValueBitmapParams) error { - _, err := q.exec(ctx, q.deleteStringAttributeValueBitmapStmt, deleteStringAttributeValueBitmap, arg.Name, arg.Value) - return err +func (q *Queries) CountDeltasSinceLastKeyframeString(ctx context.Context, arg CountDeltasSinceLastKeyframeStringParams) (int64, error) { + row := q.queryRow(ctx, q.countDeltasSinceLastKeyframeStringStmt, countDeltasSinceLastKeyframeString, arg.Name, arg.Value) + var count int64 + err := row.Scan(&count) + return count, err } -const getLastBlock = `-- name: GetLastBlock :one -SELECT block FROM last_block +const getAllNumericBitmapEntriesFrom = `-- name: GetAllNumericBitmapEntriesFrom :many +SELECT block, is_full_bitmap, bitmap FROM numeric_attributes_values_bitmaps +WHERE name = ?1 AND value = ?2 + AND block >= ?3 +ORDER BY block ASC ` -func (q *Queries) GetLastBlock(ctx context.Context) (uint64, error) { - row := q.queryRow(ctx, q.getLastBlockStmt, getLastBlock) - var block uint64 - err := row.Scan(&block) - return block, err +type GetAllNumericBitmapEntriesFromParams struct { + Name string + Value uint64 + FromBlock uint64 } -const getNumericAttributeValueBitmap = `-- name: GetNumericAttributeValueBitmap :one -SELECT bitmap FROM numeric_attributes_values_bitmaps -WHERE name = ? AND value = ? +type GetAllNumericBitmapEntriesFromRow struct { + Block uint64 + IsFullBitmap bool + Bitmap *Bitmap +} + +func (q *Queries) GetAllNumericBitmapEntriesFrom(ctx context.Context, arg GetAllNumericBitmapEntriesFromParams) ([]GetAllNumericBitmapEntriesFromRow, error) { + rows, err := q.query(ctx, q.getAllNumericBitmapEntriesFromStmt, getAllNumericBitmapEntriesFrom, arg.Name, arg.Value, arg.FromBlock) + if err != nil { + return nil, err + } + defer rows.Close() + items := []GetAllNumericBitmapEntriesFromRow{} + for rows.Next() { + var i GetAllNumericBitmapEntriesFromRow + if err := rows.Scan(&i.Block, &i.IsFullBitmap, &i.Bitmap); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const getAllStringBitmapEntriesFrom = `-- name: GetAllStringBitmapEntriesFrom :many +SELECT block, is_full_bitmap, bitmap FROM string_attributes_values_bitmaps +WHERE name = ?1 AND value = ?2 + AND block >= ?3 +ORDER BY block ASC ` -type GetNumericAttributeValueBitmapParams struct { - Name string - Value uint64 +type GetAllStringBitmapEntriesFromParams struct { + Name string + Value string + FromBlock uint64 +} + +type GetAllStringBitmapEntriesFromRow struct { + Block uint64 + IsFullBitmap bool + Bitmap *Bitmap } -func (q *Queries) GetNumericAttributeValueBitmap(ctx context.Context, arg GetNumericAttributeValueBitmapParams) (*Bitmap, error) { - row := q.queryRow(ctx, q.getNumericAttributeValueBitmapStmt, getNumericAttributeValueBitmap, arg.Name, arg.Value) - var bitmap *Bitmap - err := row.Scan(&bitmap) - return bitmap, err +func (q *Queries) GetAllStringBitmapEntriesFrom(ctx context.Context, arg GetAllStringBitmapEntriesFromParams) ([]GetAllStringBitmapEntriesFromRow, error) { + rows, err := q.query(ctx, q.getAllStringBitmapEntriesFromStmt, getAllStringBitmapEntriesFrom, arg.Name, arg.Value, arg.FromBlock) + if err != nil { + return nil, err + } + defer rows.Close() + items := []GetAllStringBitmapEntriesFromRow{} + for rows.Next() { + var i GetAllStringBitmapEntriesFromRow + if err := rows.Scan(&i.Block, &i.IsFullBitmap, &i.Bitmap); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil } -const getPayloadForEntityKey = `-- name: GetPayloadForEntityKey :one -SELECT entity_key, id, payload, content_type, string_attributes, numeric_attributes +const getCurrentPayloadForEntityKey = `-- name: GetCurrentPayloadForEntityKey :one +SELECT entity_key, id, payload, content_type, string_attributes, numeric_attributes, from_block FROM payloads -WHERE entity_key = ? +WHERE entity_key = ?1 AND to_block IS NULL ` -type GetPayloadForEntityKeyRow struct { +type GetCurrentPayloadForEntityKeyRow struct { EntityKey []byte ID uint64 Payload []byte ContentType string StringAttributes *StringAttributes NumericAttributes *NumericAttributes + FromBlock uint64 } -func (q *Queries) GetPayloadForEntityKey(ctx context.Context, entityKey []byte) (GetPayloadForEntityKeyRow, error) { - row := q.queryRow(ctx, q.getPayloadForEntityKeyStmt, getPayloadForEntityKey, entityKey) - var i GetPayloadForEntityKeyRow +func (q *Queries) GetCurrentPayloadForEntityKey(ctx context.Context, entityKey []byte) (GetCurrentPayloadForEntityKeyRow, error) { + row := q.queryRow(ctx, q.getCurrentPayloadForEntityKeyStmt, getCurrentPayloadForEntityKey, entityKey) + var i GetCurrentPayloadForEntityKeyRow err := row.Scan( &i.EntityKey, &i.ID, @@ -102,105 +178,360 @@ func (q *Queries) GetPayloadForEntityKey(ctx context.Context, entityKey []byte) &i.ContentType, &i.StringAttributes, &i.NumericAttributes, + &i.FromBlock, ) return i, err } -const getStringAttributeValueBitmap = `-- name: GetStringAttributeValueBitmap :one -SELECT bitmap FROM string_attributes_values_bitmaps -WHERE name = ? AND value = ? +const getLastBlock = `-- name: GetLastBlock :one +SELECT block FROM last_block +` + +func (q *Queries) GetLastBlock(ctx context.Context) (uint64, error) { + row := q.queryRow(ctx, q.getLastBlockStmt, getLastBlock) + var block uint64 + err := row.Scan(&block) + return block, err +} + +const getLatestNumericKeyframeBlock = `-- name: GetLatestNumericKeyframeBlock :one +SELECT COALESCE(MAX(block), -1) FROM numeric_attributes_values_bitmaps +WHERE name = ?1 AND value = ?2 AND is_full_bitmap = 1 ` -type GetStringAttributeValueBitmapParams struct { +type GetLatestNumericKeyframeBlockParams struct { + Name string + Value uint64 +} + +func (q *Queries) GetLatestNumericKeyframeBlock(ctx context.Context, arg GetLatestNumericKeyframeBlockParams) (interface{}, error) { + row := q.queryRow(ctx, q.getLatestNumericKeyframeBlockStmt, getLatestNumericKeyframeBlock, arg.Name, arg.Value) + var coalesce interface{} + err := row.Scan(&coalesce) + return coalesce, err +} + +const getLatestStringKeyframeBlock = `-- name: GetLatestStringKeyframeBlock :one +SELECT COALESCE(MAX(block), -1) FROM string_attributes_values_bitmaps +WHERE name = ?1 AND value = ?2 AND is_full_bitmap = 1 +` + +type GetLatestStringKeyframeBlockParams struct { Name string Value string } -func (q *Queries) GetStringAttributeValueBitmap(ctx context.Context, arg GetStringAttributeValueBitmapParams) (*Bitmap, error) { - row := q.queryRow(ctx, q.getStringAttributeValueBitmapStmt, getStringAttributeValueBitmap, arg.Name, arg.Value) - var bitmap *Bitmap - err := row.Scan(&bitmap) - return bitmap, err +func (q *Queries) GetLatestStringKeyframeBlock(ctx context.Context, arg GetLatestStringKeyframeBlockParams) (interface{}, error) { + row := q.queryRow(ctx, q.getLatestStringKeyframeBlockStmt, getLatestStringKeyframeBlock, arg.Name, arg.Value) + var coalesce interface{} + err := row.Scan(&coalesce) + return coalesce, err } -const upsertLastBlock = `-- name: UpsertLastBlock :exec -INSERT INTO last_block (id, block) -VALUES (1, ?) -ON CONFLICT (id) DO UPDATE SET block = EXCLUDED.block +const getNumericBitmapEntriesInRange = `-- name: GetNumericBitmapEntriesInRange :many +SELECT block, is_full_bitmap, bitmap FROM numeric_attributes_values_bitmaps +WHERE name = ?1 AND value = ?2 + AND block >= ?3 AND block <= ?4 +ORDER BY block ASC ` -func (q *Queries) UpsertLastBlock(ctx context.Context, block uint64) error { - _, err := q.exec(ctx, q.upsertLastBlockStmt, upsertLastBlock, block) - return err +type GetNumericBitmapEntriesInRangeParams struct { + Name string + Value uint64 + FromBlock uint64 + ToBlock uint64 +} + +type GetNumericBitmapEntriesInRangeRow struct { + Block uint64 + IsFullBitmap bool + Bitmap *Bitmap +} + +func (q *Queries) GetNumericBitmapEntriesInRange(ctx context.Context, arg GetNumericBitmapEntriesInRangeParams) ([]GetNumericBitmapEntriesInRangeRow, error) { + rows, err := q.query(ctx, q.getNumericBitmapEntriesInRangeStmt, getNumericBitmapEntriesInRange, + arg.Name, + arg.Value, + arg.FromBlock, + arg.ToBlock, + ) + if err != nil { + return nil, err + } + defer rows.Close() + items := []GetNumericBitmapEntriesInRangeRow{} + for rows.Next() { + var i GetNumericBitmapEntriesInRangeRow + if err := rows.Scan(&i.Block, &i.IsFullBitmap, &i.Bitmap); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const getNumericKeyframeBlockAtOrBefore = `-- name: GetNumericKeyframeBlockAtOrBefore :one +SELECT COALESCE(MAX(block), -1) FROM numeric_attributes_values_bitmaps +WHERE name = ?1 AND value = ?2 AND is_full_bitmap = 1 + AND block <= ?3 +` + +type GetNumericKeyframeBlockAtOrBeforeParams struct { + Name string + Value uint64 + TargetBlock uint64 +} + +func (q *Queries) GetNumericKeyframeBlockAtOrBefore(ctx context.Context, arg GetNumericKeyframeBlockAtOrBeforeParams) (interface{}, error) { + row := q.queryRow(ctx, q.getNumericKeyframeBlockAtOrBeforeStmt, getNumericKeyframeBlockAtOrBefore, arg.Name, arg.Value, arg.TargetBlock) + var coalesce interface{} + err := row.Scan(&coalesce) + return coalesce, err } -const upsertNumericAttributeValueBitmap = `-- name: UpsertNumericAttributeValueBitmap :exec -INSERT INTO numeric_attributes_values_bitmaps (name, value, bitmap) -VALUES (?, ?, ?) -ON CONFLICT (name, value) DO UPDATE SET bitmap = excluded.bitmap +const getStringBitmapEntriesInRange = `-- name: GetStringBitmapEntriesInRange :many +SELECT block, is_full_bitmap, bitmap FROM string_attributes_values_bitmaps +WHERE name = ?1 AND value = ?2 + AND block >= ?3 AND block <= ?4 +ORDER BY block ASC ` -type UpsertNumericAttributeValueBitmapParams struct { - Name string - Value uint64 - Bitmap *Bitmap +type GetStringBitmapEntriesInRangeParams struct { + Name string + Value string + FromBlock uint64 + ToBlock uint64 } -func (q *Queries) UpsertNumericAttributeValueBitmap(ctx context.Context, arg UpsertNumericAttributeValueBitmapParams) error { - _, err := q.exec(ctx, q.upsertNumericAttributeValueBitmapStmt, upsertNumericAttributeValueBitmap, arg.Name, arg.Value, arg.Bitmap) +type GetStringBitmapEntriesInRangeRow struct { + Block uint64 + IsFullBitmap bool + Bitmap *Bitmap +} + +func (q *Queries) GetStringBitmapEntriesInRange(ctx context.Context, arg GetStringBitmapEntriesInRangeParams) ([]GetStringBitmapEntriesInRangeRow, error) { + rows, err := q.query(ctx, q.getStringBitmapEntriesInRangeStmt, getStringBitmapEntriesInRange, + arg.Name, + arg.Value, + arg.FromBlock, + arg.ToBlock, + ) + if err != nil { + return nil, err + } + defer rows.Close() + items := []GetStringBitmapEntriesInRangeRow{} + for rows.Next() { + var i GetStringBitmapEntriesInRangeRow + if err := rows.Scan(&i.Block, &i.IsFullBitmap, &i.Bitmap); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const getStringKeyframeBlockAtOrBefore = `-- name: GetStringKeyframeBlockAtOrBefore :one +SELECT COALESCE(MAX(block), -1) FROM string_attributes_values_bitmaps +WHERE name = ?1 AND value = ?2 AND is_full_bitmap = 1 + AND block <= ?3 +` + +type GetStringKeyframeBlockAtOrBeforeParams struct { + Name string + Value string + TargetBlock uint64 +} + +func (q *Queries) GetStringKeyframeBlockAtOrBefore(ctx context.Context, arg GetStringKeyframeBlockAtOrBeforeParams) (interface{}, error) { + row := q.queryRow(ctx, q.getStringKeyframeBlockAtOrBeforeStmt, getStringKeyframeBlockAtOrBefore, arg.Name, arg.Value, arg.TargetBlock) + var coalesce interface{} + err := row.Scan(&coalesce) + return coalesce, err +} + +const insertNumericBitmapEntry = `-- name: InsertNumericBitmapEntry :exec +INSERT INTO numeric_attributes_values_bitmaps (name, value, block, is_full_bitmap, bitmap) +VALUES (?, ?, ?, ?, ?) +` + +type InsertNumericBitmapEntryParams struct { + Name string + Value uint64 + Block uint64 + IsFullBitmap bool + Bitmap *Bitmap +} + +func (q *Queries) InsertNumericBitmapEntry(ctx context.Context, arg InsertNumericBitmapEntryParams) error { + _, err := q.exec(ctx, q.insertNumericBitmapEntryStmt, insertNumericBitmapEntry, + arg.Name, + arg.Value, + arg.Block, + arg.IsFullBitmap, + arg.Bitmap, + ) return err } -const upsertPayload = `-- name: UpsertPayload :one -INSERT INTO payloads ( - entity_key, - payload, - content_type, - string_attributes, - numeric_attributes -) VALUES (?, ?, ?, ?, ?) -ON CONFLICT (entity_key) DO UPDATE SET - payload = excluded.payload, - content_type = excluded.content_type, - string_attributes = excluded.string_attributes, - numeric_attributes = excluded.numeric_attributes +const insertPayload = `-- name: InsertPayload :one +INSERT INTO payloads (entity_key, payload, content_type, string_attributes, numeric_attributes, from_block, to_block) +VALUES (?, ?, ?, ?, ?, ?, NULL) RETURNING id ` -type UpsertPayloadParams struct { +type InsertPayloadParams struct { EntityKey []byte Payload []byte ContentType string StringAttributes *StringAttributes NumericAttributes *NumericAttributes + FromBlock uint64 } -func (q *Queries) UpsertPayload(ctx context.Context, arg UpsertPayloadParams) (uint64, error) { - row := q.queryRow(ctx, q.upsertPayloadStmt, upsertPayload, +func (q *Queries) InsertPayload(ctx context.Context, arg InsertPayloadParams) (uint64, error) { + row := q.queryRow(ctx, q.insertPayloadStmt, insertPayload, arg.EntityKey, arg.Payload, arg.ContentType, arg.StringAttributes, arg.NumericAttributes, + arg.FromBlock, ) var id uint64 err := row.Scan(&id) return id, err } -const upsertStringAttributeValueBitmap = `-- name: UpsertStringAttributeValueBitmap :exec -INSERT INTO string_attributes_values_bitmaps (name, value, bitmap) -VALUES (?, ?, ?) -ON CONFLICT (name, value) DO UPDATE SET bitmap = excluded.bitmap +const insertStringBitmapEntry = `-- name: InsertStringBitmapEntry :exec +INSERT INTO string_attributes_values_bitmaps (name, value, block, is_full_bitmap, bitmap) +VALUES (?, ?, ?, ?, ?) ` -type UpsertStringAttributeValueBitmapParams struct { - Name string - Value string - Bitmap *Bitmap +type InsertStringBitmapEntryParams struct { + Name string + Value string + Block uint64 + IsFullBitmap bool + Bitmap *Bitmap +} + +func (q *Queries) InsertStringBitmapEntry(ctx context.Context, arg InsertStringBitmapEntryParams) error { + _, err := q.exec(ctx, q.insertStringBitmapEntryStmt, insertStringBitmapEntry, + arg.Name, + arg.Value, + arg.Block, + arg.IsFullBitmap, + arg.Bitmap, + ) + return err } -func (q *Queries) UpsertStringAttributeValueBitmap(ctx context.Context, arg UpsertStringAttributeValueBitmapParams) error { - _, err := q.exec(ctx, q.upsertStringAttributeValueBitmapStmt, upsertStringAttributeValueBitmap, arg.Name, arg.Value, arg.Bitmap) +const pruneNumericBitmapsBefore = `-- name: PruneNumericBitmapsBefore :exec +DELETE FROM numeric_attributes_values_bitmaps +WHERE rowid IN ( + SELECT b.rowid FROM numeric_attributes_values_bitmaps AS b + WHERE b.block < ( + SELECT COALESCE(MAX(k.block), 0) + FROM numeric_attributes_values_bitmaps AS k + WHERE k.name = b.name AND k.value = b.value + AND k.is_full_bitmap = 1 AND k.block <= ?1 + ) +) +` + +func (q *Queries) PruneNumericBitmapsBefore(ctx context.Context, pruneBlock uint64) error { + _, err := q.exec(ctx, q.pruneNumericBitmapsBeforeStmt, pruneNumericBitmapsBefore, pruneBlock) + return err +} + +const prunePayloadsBefore = `-- name: PrunePayloadsBefore :exec + +DELETE FROM payloads WHERE to_block IS NOT NULL AND to_block <= ?1 +` + +// Pruning queries +func (q *Queries) PrunePayloadsBefore(ctx context.Context, block *uint64) error { + _, err := q.exec(ctx, q.prunePayloadsBeforeStmt, prunePayloadsBefore, block) + return err +} + +const pruneStringBitmapsBefore = `-- name: PruneStringBitmapsBefore :exec +DELETE FROM string_attributes_values_bitmaps +WHERE rowid IN ( + SELECT b.rowid FROM string_attributes_values_bitmaps AS b + WHERE b.block < ( + SELECT COALESCE(MAX(k.block), 0) + FROM string_attributes_values_bitmaps AS k + WHERE k.name = b.name AND k.value = b.value + AND k.is_full_bitmap = 1 AND k.block <= ?1 + ) +) +` + +func (q *Queries) PruneStringBitmapsBefore(ctx context.Context, pruneBlock uint64) error { + _, err := q.exec(ctx, q.pruneStringBitmapsBeforeStmt, pruneStringBitmapsBefore, pruneBlock) + return err +} + +const reorgDeleteNewPayloads = `-- name: ReorgDeleteNewPayloads :exec +DELETE FROM payloads WHERE from_block > ?1 +` + +func (q *Queries) ReorgDeleteNewPayloads(ctx context.Context, block uint64) error { + _, err := q.exec(ctx, q.reorgDeleteNewPayloadsStmt, reorgDeleteNewPayloads, block) + return err +} + +const reorgDeleteNumericBitmaps = `-- name: ReorgDeleteNumericBitmaps :exec +DELETE FROM numeric_attributes_values_bitmaps WHERE block > ?1 +` + +func (q *Queries) ReorgDeleteNumericBitmaps(ctx context.Context, block uint64) error { + _, err := q.exec(ctx, q.reorgDeleteNumericBitmapsStmt, reorgDeleteNumericBitmaps, block) + return err +} + +const reorgDeleteStringBitmaps = `-- name: ReorgDeleteStringBitmaps :exec +DELETE FROM string_attributes_values_bitmaps WHERE block > ?1 +` + +func (q *Queries) ReorgDeleteStringBitmaps(ctx context.Context, block uint64) error { + _, err := q.exec(ctx, q.reorgDeleteStringBitmapsStmt, reorgDeleteStringBitmaps, block) + return err +} + +const reorgReopenPayloads = `-- name: ReorgReopenPayloads :exec + +UPDATE payloads SET to_block = NULL +WHERE from_block <= ?1 AND to_block IS NOT NULL AND to_block > ?1 +` + +// Reorg queries +func (q *Queries) ReorgReopenPayloads(ctx context.Context, block uint64) error { + _, err := q.exec(ctx, q.reorgReopenPayloadsStmt, reorgReopenPayloads, block) + return err +} + +const upsertLastBlock = `-- name: UpsertLastBlock :exec +INSERT INTO last_block (id, block) +VALUES (1, ?) +ON CONFLICT (id) DO UPDATE SET block = EXCLUDED.block +` + +func (q *Queries) UpsertLastBlock(ctx context.Context, block uint64) error { + _, err := q.exec(ctx, q.upsertLastBlockStmt, upsertLastBlock, block) return err } diff --git a/store/queries/evals.sql b/store/queries/evals.sql index c696f4a..4c260b0 100644 --- a/store/queries/evals.sql +++ b/store/queries/evals.sql @@ -1,75 +1,97 @@ --- name: EvaluateAll :many -SELECT id FROM payloads -ORDER BY id DESC; - --- name: EvaluateStringAttributeValueEqual :one -SELECT bitmap FROM string_attributes_values_bitmaps -WHERE name = sqlc.arg(name) AND value = sqlc.arg(value); - --- name: EvaluateNumericAttributeValueEqual :one -SELECT bitmap FROM numeric_attributes_values_bitmaps -WHERE name = sqlc.arg(name) AND value = sqlc.arg(value); - --- name: EvaluateStringAttributeValueNotEqual :many -SELECT bitmap FROM string_attributes_values_bitmaps -WHERE name = sqlc.arg(name) AND value != sqlc.arg(value); - --- name: EvaluateNumericAttributeValueNotEqual :many -SELECT bitmap FROM numeric_attributes_values_bitmaps -WHERE name = sqlc.arg(name) AND value != sqlc.arg(value); - --- name: EvaluateStringAttributeValueLowerThan :many -SELECT bitmap FROM string_attributes_values_bitmaps -WHERE name = sqlc.arg(name) AND value < sqlc.arg(value); - --- name: EvaluateStringAttributeValueGreaterThan :many -SELECT bitmap FROM string_attributes_values_bitmaps -WHERE name = sqlc.arg(name) AND value > sqlc.arg(value); - --- name: EvaluateStringAttributeValueLessOrEqualThan :many -SELECT bitmap FROM string_attributes_values_bitmaps -WHERE name = sqlc.arg(name) AND value <= sqlc.arg(value); - --- name: EvaluateStringAttributeValueGreaterOrEqualThan :many -SELECT bitmap FROM string_attributes_values_bitmaps -WHERE name = sqlc.arg(name) AND value >= sqlc.arg(value); - --- name: EvaluateStringAttributeValueGlob :many -SELECT bitmap FROM string_attributes_values_bitmaps -WHERE name = sqlc.arg(name) AND value GLOB sqlc.arg(value); - --- name: EvaluateStringAttributeValueNotGlob :many -SELECT bitmap FROM string_attributes_values_bitmaps -WHERE name = sqlc.arg(name) AND value NOT GLOB sqlc.arg(value); - --- name: EvaluateStringAttributeValueNotInclusion :many -SELECT bitmap FROM string_attributes_values_bitmaps -WHERE name = sqlc.arg(name) AND value NOT IN (sqlc.Slice('values')); - --- name: EvaluateStringAttributeValueInclusion :many -SELECT bitmap FROM string_attributes_values_bitmaps -WHERE name = sqlc.arg(name) AND value IN (sqlc.Slice('values')); - --- name: EvaluateNumericAttributeValueLowerThan :many -SELECT bitmap FROM numeric_attributes_values_bitmaps -WHERE name = sqlc.arg(name) AND value < sqlc.arg(value); - --- name: EvaluateNumericAttributeValueGreaterThan :many -SELECT bitmap FROM numeric_attributes_values_bitmaps -WHERE name = sqlc.arg(name) AND value > sqlc.arg(value); - --- name: EvaluateNumericAttributeValueLessOrEqualThan :many -SELECT bitmap FROM numeric_attributes_values_bitmaps -WHERE name = sqlc.arg(name) AND value <= sqlc.arg(value); - --- name: EvaluateNumericAttributeValueGreaterOrEqualThan :many -SELECT bitmap FROM numeric_attributes_values_bitmaps -WHERE name = sqlc.arg(name) AND value >= sqlc.arg(value); - --- name: EvaluateNumericAttributeValueInclusion :many -SELECT bitmap FROM numeric_attributes_values_bitmaps -WHERE name = sqlc.arg(name) AND value IN (sqlc.Slice('values')); - --- name: EvaluateNumericAttributeValueNotInclusion :many -SELECT bitmap FROM numeric_attributes_values_bitmaps -WHERE name = sqlc.arg(name) AND value NOT IN (sqlc.Slice('values')); +-- Value-listing queries for query evaluation. +-- Each returns distinct values matching a condition, filtered by block. +-- The Go layer then reconstructs bitmaps for each matching value via XOR chains. + +-- String attribute value queries + +-- name: GetMatchingStringValuesEqual :many +SELECT DISTINCT value FROM string_attributes_values_bitmaps +WHERE name = sqlc.arg(name) AND value = sqlc.arg(value) + AND block <= sqlc.arg(target_block); + +-- name: GetMatchingStringValuesNotEqual :many +SELECT DISTINCT value FROM string_attributes_values_bitmaps +WHERE name = sqlc.arg(name) AND value != sqlc.arg(value) + AND block <= sqlc.arg(target_block); + +-- name: GetMatchingStringValuesLessThan :many +SELECT DISTINCT value FROM string_attributes_values_bitmaps +WHERE name = sqlc.arg(name) AND value < sqlc.arg(value) + AND block <= sqlc.arg(target_block); + +-- name: GetMatchingStringValuesGreaterThan :many +SELECT DISTINCT value FROM string_attributes_values_bitmaps +WHERE name = sqlc.arg(name) AND value > sqlc.arg(value) + AND block <= sqlc.arg(target_block); + +-- name: GetMatchingStringValuesLessOrEqualThan :many +SELECT DISTINCT value FROM string_attributes_values_bitmaps +WHERE name = sqlc.arg(name) AND value <= sqlc.arg(value) + AND block <= sqlc.arg(target_block); + +-- name: GetMatchingStringValuesGreaterOrEqualThan :many +SELECT DISTINCT value FROM string_attributes_values_bitmaps +WHERE name = sqlc.arg(name) AND value >= sqlc.arg(value) + AND block <= sqlc.arg(target_block); + +-- name: GetMatchingStringValuesGlob :many +SELECT DISTINCT value FROM string_attributes_values_bitmaps +WHERE name = sqlc.arg(name) AND value GLOB sqlc.arg(value) + AND block <= sqlc.arg(target_block); + +-- name: GetMatchingStringValuesNotGlob :many +SELECT DISTINCT value FROM string_attributes_values_bitmaps +WHERE name = sqlc.arg(name) AND value NOT GLOB sqlc.arg(value) + AND block <= sqlc.arg(target_block); + +-- name: GetMatchingStringValuesInclusion :many +SELECT DISTINCT value FROM string_attributes_values_bitmaps +WHERE name = sqlc.arg(name) AND value IN (sqlc.slice('values')) + AND block <= sqlc.arg(target_block); + +-- name: GetMatchingStringValuesNotInclusion :many +SELECT DISTINCT value FROM string_attributes_values_bitmaps +WHERE name = sqlc.arg(name) AND value NOT IN (sqlc.slice('values')) + AND block <= sqlc.arg(target_block); + +-- Numeric attribute value queries + +-- name: GetMatchingNumericValuesEqual :many +SELECT DISTINCT value FROM numeric_attributes_values_bitmaps +WHERE name = sqlc.arg(name) AND value = sqlc.arg(value) + AND block <= sqlc.arg(target_block); + +-- name: GetMatchingNumericValuesNotEqual :many +SELECT DISTINCT value FROM numeric_attributes_values_bitmaps +WHERE name = sqlc.arg(name) AND value != sqlc.arg(value) + AND block <= sqlc.arg(target_block); + +-- name: GetMatchingNumericValuesLessThan :many +SELECT DISTINCT value FROM numeric_attributes_values_bitmaps +WHERE name = sqlc.arg(name) AND value < sqlc.arg(value) + AND block <= sqlc.arg(target_block); + +-- name: GetMatchingNumericValuesGreaterThan :many +SELECT DISTINCT value FROM numeric_attributes_values_bitmaps +WHERE name = sqlc.arg(name) AND value > sqlc.arg(value) + AND block <= sqlc.arg(target_block); + +-- name: GetMatchingNumericValuesLessOrEqualThan :many +SELECT DISTINCT value FROM numeric_attributes_values_bitmaps +WHERE name = sqlc.arg(name) AND value <= sqlc.arg(value) + AND block <= sqlc.arg(target_block); + +-- name: GetMatchingNumericValuesGreaterOrEqualThan :many +SELECT DISTINCT value FROM numeric_attributes_values_bitmaps +WHERE name = sqlc.arg(name) AND value >= sqlc.arg(value) + AND block <= sqlc.arg(target_block); + +-- name: GetMatchingNumericValuesInclusion :many +SELECT DISTINCT value FROM numeric_attributes_values_bitmaps +WHERE name = sqlc.arg(name) AND value IN (sqlc.slice('values')) + AND block <= sqlc.arg(target_block); + +-- name: GetMatchingNumericValuesNotInclusion :many +SELECT DISTINCT value FROM numeric_attributes_values_bitmaps +WHERE name = sqlc.arg(name) AND value NOT IN (sqlc.slice('values')) + AND block <= sqlc.arg(target_block); diff --git a/store/queries/queries.sql b/store/queries/queries.sql index 70ff4c5..2583b58 100644 --- a/store/queries/queries.sql +++ b/store/queries/queries.sql @@ -1,52 +1,84 @@ --- name: UpsertPayload :one -INSERT INTO payloads ( - entity_key, - payload, - content_type, - string_attributes, - numeric_attributes -) VALUES (?, ?, ?, ?, ?) -ON CONFLICT (entity_key) DO UPDATE SET - payload = excluded.payload, - content_type = excluded.content_type, - string_attributes = excluded.string_attributes, - numeric_attributes = excluded.numeric_attributes -RETURNING id; +-- name: ClosePayloadVersion :exec +UPDATE payloads SET to_block = sqlc.arg(block) +WHERE entity_key = sqlc.arg(entity_key) AND to_block IS NULL; --- name: DeletePayloadForEntityKey :exec -DELETE FROM payloads -WHERE entity_key = ?; +-- name: InsertPayload :one +INSERT INTO payloads (entity_key, payload, content_type, string_attributes, numeric_attributes, from_block, to_block) +VALUES (?, ?, ?, ?, ?, ?, NULL) +RETURNING id; --- name: GetPayloadForEntityKey :one -SELECT entity_key, id, payload, content_type, string_attributes, numeric_attributes +-- name: GetCurrentPayloadForEntityKey :one +SELECT entity_key, id, payload, content_type, string_attributes, numeric_attributes, from_block FROM payloads -WHERE entity_key = ?; +WHERE entity_key = sqlc.arg(entity_key) AND to_block IS NULL; --- name: UpsertStringAttributeValueBitmap :exec -INSERT INTO string_attributes_values_bitmaps (name, value, bitmap) -VALUES (?, ?, ?) -ON CONFLICT (name, value) DO UPDATE SET bitmap = excluded.bitmap; +-- name: InsertStringBitmapEntry :exec +INSERT INTO string_attributes_values_bitmaps (name, value, block, is_full_bitmap, bitmap) +VALUES (?, ?, ?, ?, ?); --- name: DeleteStringAttributeValueBitmap :exec -DELETE FROM string_attributes_values_bitmaps -WHERE name = ? AND value = ?; +-- name: InsertNumericBitmapEntry :exec +INSERT INTO numeric_attributes_values_bitmaps (name, value, block, is_full_bitmap, bitmap) +VALUES (?, ?, ?, ?, ?); --- name: GetStringAttributeValueBitmap :one -SELECT bitmap FROM string_attributes_values_bitmaps -WHERE name = ? AND value = ?; +-- name: GetLatestStringKeyframeBlock :one +SELECT COALESCE(MAX(block), -1) FROM string_attributes_values_bitmaps +WHERE name = sqlc.arg(name) AND value = sqlc.arg(value) AND is_full_bitmap = 1; --- name: UpsertNumericAttributeValueBitmap :exec -INSERT INTO numeric_attributes_values_bitmaps (name, value, bitmap) -VALUES (?, ?, ?) -ON CONFLICT (name, value) DO UPDATE SET bitmap = excluded.bitmap; +-- name: GetLatestNumericKeyframeBlock :one +SELECT COALESCE(MAX(block), -1) FROM numeric_attributes_values_bitmaps +WHERE name = sqlc.arg(name) AND value = sqlc.arg(value) AND is_full_bitmap = 1; --- name: DeleteNumericAttributeValueBitmap :exec -DELETE FROM numeric_attributes_values_bitmaps -WHERE name = ? AND value = ?; +-- name: GetStringKeyframeBlockAtOrBefore :one +SELECT COALESCE(MAX(block), -1) FROM string_attributes_values_bitmaps +WHERE name = sqlc.arg(name) AND value = sqlc.arg(value) AND is_full_bitmap = 1 + AND block <= sqlc.arg(target_block); + +-- name: GetNumericKeyframeBlockAtOrBefore :one +SELECT COALESCE(MAX(block), -1) FROM numeric_attributes_values_bitmaps +WHERE name = sqlc.arg(name) AND value = sqlc.arg(value) AND is_full_bitmap = 1 + AND block <= sqlc.arg(target_block); + +-- name: GetStringBitmapEntriesInRange :many +SELECT block, is_full_bitmap, bitmap FROM string_attributes_values_bitmaps +WHERE name = sqlc.arg(name) AND value = sqlc.arg(value) + AND block >= sqlc.arg(from_block) AND block <= sqlc.arg(to_block) +ORDER BY block ASC; --- name: GetNumericAttributeValueBitmap :one -SELECT bitmap FROM numeric_attributes_values_bitmaps -WHERE name = ? AND value = ?; +-- name: GetNumericBitmapEntriesInRange :many +SELECT block, is_full_bitmap, bitmap FROM numeric_attributes_values_bitmaps +WHERE name = sqlc.arg(name) AND value = sqlc.arg(value) + AND block >= sqlc.arg(from_block) AND block <= sqlc.arg(to_block) +ORDER BY block ASC; + +-- name: GetAllStringBitmapEntriesFrom :many +SELECT block, is_full_bitmap, bitmap FROM string_attributes_values_bitmaps +WHERE name = sqlc.arg(name) AND value = sqlc.arg(value) + AND block >= sqlc.arg(from_block) +ORDER BY block ASC; + +-- name: GetAllNumericBitmapEntriesFrom :many +SELECT block, is_full_bitmap, bitmap FROM numeric_attributes_values_bitmaps +WHERE name = sqlc.arg(name) AND value = sqlc.arg(value) + AND block >= sqlc.arg(from_block) +ORDER BY block ASC; + +-- name: CountDeltasSinceLastKeyframeString :one +SELECT COUNT(*) FROM string_attributes_values_bitmaps AS outer_s +WHERE outer_s.name = sqlc.arg(name) AND outer_s.value = sqlc.arg(value) AND outer_s.is_full_bitmap = 0 + AND outer_s.block > ( + SELECT COALESCE(MAX(inner_s.block), -1) + FROM string_attributes_values_bitmaps AS inner_s + WHERE inner_s.name = sqlc.arg(name) AND inner_s.value = sqlc.arg(value) AND inner_s.is_full_bitmap = 1 + ); + +-- name: CountDeltasSinceLastKeyframeNumeric :one +SELECT COUNT(*) FROM numeric_attributes_values_bitmaps AS outer_n +WHERE outer_n.name = sqlc.arg(name) AND outer_n.value = sqlc.arg(value) AND outer_n.is_full_bitmap = 0 + AND outer_n.block > ( + SELECT COALESCE(MAX(inner_n.block), -1) + FROM numeric_attributes_values_bitmaps AS inner_n + WHERE inner_n.name = sqlc.arg(name) AND inner_n.value = sqlc.arg(value) AND inner_n.is_full_bitmap = 1 + ); -- name: UpsertLastBlock :exec INSERT INTO last_block (id, block) @@ -55,3 +87,47 @@ ON CONFLICT (id) DO UPDATE SET block = EXCLUDED.block; -- name: GetLastBlock :one SELECT block FROM last_block; + +-- Pruning queries + +-- name: PrunePayloadsBefore :exec +DELETE FROM payloads WHERE to_block IS NOT NULL AND to_block <= sqlc.arg(block); + +-- name: PruneStringBitmapsBefore :exec +DELETE FROM string_attributes_values_bitmaps +WHERE rowid IN ( + SELECT b.rowid FROM string_attributes_values_bitmaps AS b + WHERE b.block < ( + SELECT COALESCE(MAX(k.block), 0) + FROM string_attributes_values_bitmaps AS k + WHERE k.name = b.name AND k.value = b.value + AND k.is_full_bitmap = 1 AND k.block <= sqlc.arg(prune_block) + ) +); + +-- name: PruneNumericBitmapsBefore :exec +DELETE FROM numeric_attributes_values_bitmaps +WHERE rowid IN ( + SELECT b.rowid FROM numeric_attributes_values_bitmaps AS b + WHERE b.block < ( + SELECT COALESCE(MAX(k.block), 0) + FROM numeric_attributes_values_bitmaps AS k + WHERE k.name = b.name AND k.value = b.value + AND k.is_full_bitmap = 1 AND k.block <= sqlc.arg(prune_block) + ) +); + +-- Reorg queries + +-- name: ReorgReopenPayloads :exec +UPDATE payloads SET to_block = NULL +WHERE from_block <= sqlc.arg(block) AND to_block IS NOT NULL AND to_block > sqlc.arg(block); + +-- name: ReorgDeleteNewPayloads :exec +DELETE FROM payloads WHERE from_block > sqlc.arg(block); + +-- name: ReorgDeleteStringBitmaps :exec +DELETE FROM string_attributes_values_bitmaps WHERE block > sqlc.arg(block); + +-- name: ReorgDeleteNumericBitmaps :exec +DELETE FROM numeric_attributes_values_bitmaps WHERE block > sqlc.arg(block); diff --git a/store/queries/retrieve.sql b/store/queries/retrieve.sql index 98bf897..e66a573 100644 --- a/store/queries/retrieve.sql +++ b/store/queries/retrieve.sql @@ -4,5 +4,15 @@ FROM payloads WHERE id IN (sqlc.slice(ids)) ORDER BY id DESC; +-- name: EvaluateAllCurrent :many +SELECT id FROM payloads +WHERE to_block IS NULL +ORDER BY id DESC; + +-- name: EvaluateAllAtBlock :many +SELECT id FROM payloads +WHERE from_block <= sqlc.arg(block) AND (to_block IS NULL OR to_block > sqlc.arg(block)) +ORDER BY id DESC; + -- name: GetNumberOfEntities :one -SELECT COUNT(*) FROM payloads; \ No newline at end of file +SELECT COUNT(*) FROM payloads WHERE to_block IS NULL; diff --git a/store/retrieve.sql.go b/store/retrieve.sql.go index 108badb..d1dd3a5 100644 --- a/store/retrieve.sql.go +++ b/store/retrieve.sql.go @@ -10,8 +10,66 @@ import ( "strings" ) +const evaluateAllAtBlock = `-- name: EvaluateAllAtBlock :many +SELECT id FROM payloads +WHERE from_block <= ?1 AND (to_block IS NULL OR to_block > ?1) +ORDER BY id DESC +` + +func (q *Queries) EvaluateAllAtBlock(ctx context.Context, block uint64) ([]uint64, error) { + rows, err := q.query(ctx, q.evaluateAllAtBlockStmt, evaluateAllAtBlock, block) + if err != nil { + return nil, err + } + defer rows.Close() + items := []uint64{} + for rows.Next() { + var id uint64 + if err := rows.Scan(&id); err != nil { + return nil, err + } + items = append(items, id) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const evaluateAllCurrent = `-- name: EvaluateAllCurrent :many +SELECT id FROM payloads +WHERE to_block IS NULL +ORDER BY id DESC +` + +func (q *Queries) EvaluateAllCurrent(ctx context.Context) ([]uint64, error) { + rows, err := q.query(ctx, q.evaluateAllCurrentStmt, evaluateAllCurrent) + if err != nil { + return nil, err + } + defer rows.Close() + items := []uint64{} + for rows.Next() { + var id uint64 + if err := rows.Scan(&id); err != nil { + return nil, err + } + items = append(items, id) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const getNumberOfEntities = `-- name: GetNumberOfEntities :one -SELECT COUNT(*) FROM payloads +SELECT COUNT(*) FROM payloads WHERE to_block IS NULL ` func (q *Queries) GetNumberOfEntities(ctx context.Context) (int64, error) { diff --git a/store/schema/000002_bitemporality.down.sql b/store/schema/000002_bitemporality.down.sql new file mode 100644 index 0000000..d57f05e --- /dev/null +++ b/store/schema/000002_bitemporality.down.sql @@ -0,0 +1,64 @@ +-- Rollback bitemporality: recreate original tables without temporal columns. +-- WARNING: This discards all historical data, keeping only current versions. + +-- 1. Payloads: remove from_block, to_block; restore original unique index +CREATE TABLE payloads_old ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + entity_key BLOB NOT NULL, + payload BLOB NOT NULL, + content_type TEXT NOT NULL DEFAULT '', + string_attributes TEXT NOT NULL DEFAULT '{}', + numeric_attributes TEXT NOT NULL DEFAULT '{}' +); + +INSERT INTO payloads_old (id, entity_key, payload, content_type, string_attributes, numeric_attributes) +SELECT id, entity_key, payload, content_type, string_attributes, numeric_attributes +FROM payloads +WHERE to_block IS NULL; + +DROP TABLE payloads; +ALTER TABLE payloads_old RENAME TO payloads; + +CREATE UNIQUE INDEX payloads_entity_key_index ON payloads (entity_key); + +-- 2. String attribute bitmaps: keep only the latest keyframe per (name, value) +CREATE TABLE string_attributes_values_bitmaps_old ( + name TEXT NOT NULL, + value TEXT NOT NULL, + bitmap BLOB, + PRIMARY KEY (name, value) +); + +INSERT INTO string_attributes_values_bitmaps_old (name, value, bitmap) +SELECT s.name, s.value, s.bitmap +FROM string_attributes_values_bitmaps s +INNER JOIN ( + SELECT name, value, MAX(block) AS max_block + FROM string_attributes_values_bitmaps + WHERE is_full_bitmap = 1 + GROUP BY name, value +) latest ON s.name = latest.name AND s.value = latest.value AND s.block = latest.max_block; + +DROP TABLE string_attributes_values_bitmaps; +ALTER TABLE string_attributes_values_bitmaps_old RENAME TO string_attributes_values_bitmaps; + +-- 3. Numeric attribute bitmaps: keep only the latest keyframe per (name, value) +CREATE TABLE numeric_attributes_values_bitmaps_old ( + name TEXT NOT NULL, + value INTEGER NOT NULL, + bitmap BLOB, + PRIMARY KEY (name, value) +); + +INSERT INTO numeric_attributes_values_bitmaps_old (name, value, bitmap) +SELECT n.name, n.value, n.bitmap +FROM numeric_attributes_values_bitmaps n +INNER JOIN ( + SELECT name, value, MAX(block) AS max_block + FROM numeric_attributes_values_bitmaps + WHERE is_full_bitmap = 1 + GROUP BY name, value +) latest ON n.name = latest.name AND n.value = latest.value AND n.block = latest.max_block; + +DROP TABLE numeric_attributes_values_bitmaps; +ALTER TABLE numeric_attributes_values_bitmaps_old RENAME TO numeric_attributes_values_bitmaps; diff --git a/store/schema/000002_bitemporality.up.sql b/store/schema/000002_bitemporality.up.sql new file mode 100644 index 0000000..fc4f4d9 --- /dev/null +++ b/store/schema/000002_bitemporality.up.sql @@ -0,0 +1,60 @@ +-- Bitemporality migration: add temporal columns to payloads and bitmap tables. +-- SQLite does not support ALTER TABLE to change primary keys, so we recreate tables. + +-- 1. Payloads: add from_block and to_block columns +CREATE TABLE payloads_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + entity_key BLOB NOT NULL, + payload BLOB NOT NULL, + content_type TEXT NOT NULL DEFAULT '', + string_attributes TEXT NOT NULL DEFAULT '{}', + numeric_attributes TEXT NOT NULL DEFAULT '{}', + from_block INTEGER NOT NULL DEFAULT 0, + to_block INTEGER -- NULL means "current" / still active +); + +INSERT INTO payloads_new (id, entity_key, payload, content_type, string_attributes, numeric_attributes, from_block, to_block) +SELECT id, entity_key, payload, content_type, string_attributes, numeric_attributes, 0, NULL +FROM payloads; + +DROP TABLE payloads; +ALTER TABLE payloads_new RENAME TO payloads; + +-- Partial unique index: at most one current (to_block IS NULL) row per entity_key +CREATE UNIQUE INDEX payloads_entity_key_current ON payloads (entity_key) WHERE to_block IS NULL; +-- Index for historical lookups +CREATE INDEX payloads_entity_key_blocks ON payloads (entity_key, from_block, to_block); + +-- 2. String attribute bitmaps: add block and is_full_bitmap columns, change PK +CREATE TABLE string_attributes_values_bitmaps_new ( + name TEXT NOT NULL, + value TEXT NOT NULL, + block INTEGER NOT NULL DEFAULT 0, + is_full_bitmap INTEGER NOT NULL DEFAULT 1, + bitmap BLOB, + PRIMARY KEY (name, value, block) +); + +INSERT INTO string_attributes_values_bitmaps_new (name, value, block, is_full_bitmap, bitmap) +SELECT name, value, 0, 1, bitmap +FROM string_attributes_values_bitmaps; + +DROP TABLE string_attributes_values_bitmaps; +ALTER TABLE string_attributes_values_bitmaps_new RENAME TO string_attributes_values_bitmaps; + +-- 3. Numeric attribute bitmaps: add block and is_full_bitmap columns, change PK +CREATE TABLE numeric_attributes_values_bitmaps_new ( + name TEXT NOT NULL, + value INTEGER NOT NULL, + block INTEGER NOT NULL DEFAULT 0, + is_full_bitmap INTEGER NOT NULL DEFAULT 1, + bitmap BLOB, + PRIMARY KEY (name, value, block) +); + +INSERT INTO numeric_attributes_values_bitmaps_new (name, value, block, is_full_bitmap, bitmap) +SELECT name, value, 0, 1, bitmap +FROM numeric_attributes_values_bitmaps; + +DROP TABLE numeric_attributes_values_bitmaps; +ALTER TABLE numeric_attributes_values_bitmaps_new RENAME TO numeric_attributes_values_bitmaps; diff --git a/store/sqlc.yaml b/store/sqlc.yaml index 47aa2ae..8a7d6ca 100644 --- a/store/sqlc.yaml +++ b/store/sqlc.yaml @@ -31,7 +31,21 @@ sql: type: "Bitmap" pointer: true - column: "numeric_attributes_values_bitmaps.bitmap" - go_type: + go_type: import: "" type: "Bitmap" pointer: true + - column: "payloads.from_block" + go_type: "uint64" + - column: "payloads.to_block" + go_type: + type: "uint64" + pointer: true + - column: "string_attributes_values_bitmaps.block" + go_type: "uint64" + - column: "string_attributes_values_bitmaps.is_full_bitmap" + go_type: "bool" + - column: "numeric_attributes_values_bitmaps.block" + go_type: "uint64" + - column: "numeric_attributes_values_bitmaps.is_full_bitmap" + go_type: "bool" diff --git a/update_index.go b/update_index.go new file mode 100644 index 0000000..348f485 --- /dev/null +++ b/update_index.go @@ -0,0 +1,52 @@ +package sqlitebitmapstore + +import ( + "context" + "database/sql" + "fmt" + "time" + + "github.com/Arkiv-Network/sqlite-bitmap-store/store" +) + +func (s *SQLiteStore) UpdateIndex(ctx context.Context, block uint64, update func(cache *bitmapCache, lastBlockFromDB uint64) error) error { + + tx, err := s.writePool.BeginTx(ctx, &sql.TxOptions{ + Isolation: sql.LevelSerializable, + ReadOnly: false, + }) + if err != nil { + return fmt.Errorf("failed to begin transaction: %w", err) + } + defer tx.Rollback() + + st := store.New(tx) + + lastBlockFromDB, err := st.GetLastBlock(ctx) + if err != nil { + return fmt.Errorf("failed to get last block from database: %w", err) + } + + cache := newBitmapCache(st, block) + + startTime := time.Now() + + err = update(cache, lastBlockFromDB) + if err != nil { + return fmt.Errorf("failed to update index: %w", err) + } + + err = cache.Flush(ctx) + if err != nil { + return fmt.Errorf("failed to flush cache: %w", err) + } + + err = tx.Commit() + if err != nil { + return fmt.Errorf("failed to commit transaction: %w", err) + } + + s.log.Info("index updated", "processingTime", time.Since(startTime).Milliseconds()) + + return nil +} From 738d89e7778501de0a3b79d4eaa0694ca9b0b887 Mon Sep 17 00:00:00 2001 From: Dragan Milic Date: Wed, 11 Mar 2026 11:04:19 +0100 Subject: [PATCH 3/5] added pprof to cmd/load-from-tar --- cmd/load-from-tar/main.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/cmd/load-from-tar/main.go b/cmd/load-from-tar/main.go index 345e406..c956091 100644 --- a/cmd/load-from-tar/main.go +++ b/cmd/load-from-tar/main.go @@ -5,6 +5,8 @@ import ( "fmt" "log" "log/slog" + "net/http" + _ "net/http/pprof" "os" "os/signal" "syscall" @@ -20,7 +22,8 @@ func main() { logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) cfg := struct { - dbPath string + dbPath string + pprofAddr string }{} app := &cli.App{ @@ -33,9 +36,22 @@ func main() { Destination: &cfg.dbPath, EnvVars: []string{"DB_PATH"}, }, + &cli.StringFlag{ + Name: "pprof-addr", + Value: "localhost:6060", + Destination: &cfg.pprofAddr, + EnvVars: []string{"PPROF_ADDR"}, + }, }, Action: func(c *cli.Context) error { + go func() { + logger.Info("starting pprof server", "addr", cfg.pprofAddr) + if err := http.ListenAndServe(cfg.pprofAddr, nil); err != nil { + logger.Error("pprof server failed", "error", err) + } + }() + tarFileName := c.Args().First() if tarFileName == "" { From 9ff6f969a3d2d740521c92a8c4185c84c8d641bf Mon Sep 17 00:00:00 2001 From: Dragan Milic Date: Wed, 11 Mar 2026 12:11:16 +0100 Subject: [PATCH 4/5] optimize bitmap cache --- bitmap_cache.go | 116 ++++++++++++++++++++++++++++++++---------------- sqlitestore.go | 6 ++- 2 files changed, 81 insertions(+), 41 deletions(-) diff --git a/bitmap_cache.go b/bitmap_cache.go index 12f7735..1fff047 100644 --- a/bitmap_cache.go +++ b/bitmap_cache.go @@ -24,16 +24,22 @@ type bitmapCache struct { // Old bitmaps (state before modifications) for computing deltas. stringOldBitmaps map[nameValue[string]]*store.Bitmap numericOldBitmaps map[nameValue[uint64]]*store.Bitmap + + // In-memory delta counts to avoid querying CountDeltasSinceLastKeyframe every Flush. + stringDeltaCounts map[nameValue[string]]int64 + numericDeltaCounts map[nameValue[uint64]]int64 } func newBitmapCache(st *store.Queries, block uint64) *bitmapCache { return &bitmapCache{ - st: st, - block: block, - stringBitmaps: make(map[nameValue[string]]*store.Bitmap), - numericBitmaps: make(map[nameValue[uint64]]*store.Bitmap), - stringOldBitmaps: make(map[nameValue[string]]*store.Bitmap), - numericOldBitmaps: make(map[nameValue[uint64]]*store.Bitmap), + st: st, + block: block, + stringBitmaps: make(map[nameValue[string]]*store.Bitmap), + numericBitmaps: make(map[nameValue[uint64]]*store.Bitmap), + stringOldBitmaps: make(map[nameValue[string]]*store.Bitmap), + numericOldBitmaps: make(map[nameValue[uint64]]*store.Bitmap), + stringDeltaCounts: make(map[nameValue[string]]int64), + numericDeltaCounts: make(map[nameValue[uint64]]int64), } } @@ -44,16 +50,53 @@ func cloneBitmap(bm *store.Bitmap) *store.Bitmap { return &store.Bitmap{Bitmap: bm.Bitmap.Clone()} } +func (c *bitmapCache) loadStringBitmap(ctx context.Context, k nameValue[string]) (*store.Bitmap, error) { + bitmap, err := c.st.ReconstructLatestStringBitmap(ctx, k.name, k.value) + if err != nil { + return nil, fmt.Errorf("failed to reconstruct string bitmap %q=%q: %w", k.name, k.value, err) + } + c.stringOldBitmaps[k] = cloneBitmap(bitmap) + c.stringBitmaps[k] = bitmap + + // Seed delta count from DB so we know when next keyframe is due. + deltaCount, err := c.st.CountDeltasSinceLastKeyframeString(ctx, store.CountDeltasSinceLastKeyframeStringParams{ + Name: k.name, Value: k.value, + }) + if err != nil { + return nil, fmt.Errorf("failed to count deltas for string %q=%q: %w", k.name, k.value, err) + } + c.stringDeltaCounts[k] = deltaCount + + return bitmap, nil +} + +func (c *bitmapCache) loadNumericBitmap(ctx context.Context, k nameValue[uint64]) (*store.Bitmap, error) { + bitmap, err := c.st.ReconstructLatestNumericBitmap(ctx, k.name, k.value) + if err != nil { + return nil, fmt.Errorf("failed to reconstruct numeric bitmap %q=%d: %w", k.name, k.value, err) + } + c.numericOldBitmaps[k] = cloneBitmap(bitmap) + c.numericBitmaps[k] = bitmap + + deltaCount, err := c.st.CountDeltasSinceLastKeyframeNumeric(ctx, store.CountDeltasSinceLastKeyframeNumericParams{ + Name: k.name, Value: k.value, + }) + if err != nil { + return nil, fmt.Errorf("failed to count deltas for numeric %q=%d: %w", k.name, k.value, err) + } + c.numericDeltaCounts[k] = deltaCount + + return bitmap, nil +} + func (c *bitmapCache) AddToStringBitmap(ctx context.Context, name string, value string, id uint64) (err error) { k := nameValue[string]{name: name, value: value} bitmap, ok := c.stringBitmaps[k] if !ok { - bitmap, err = c.st.ReconstructLatestStringBitmap(ctx, name, value) + bitmap, err = c.loadStringBitmap(ctx, k) if err != nil { - return fmt.Errorf("failed to reconstruct string bitmap %q=%q: %w", name, value, err) + return err } - c.stringOldBitmaps[k] = cloneBitmap(bitmap) - c.stringBitmaps[k] = bitmap } bitmap.Add(id) @@ -64,12 +107,10 @@ func (c *bitmapCache) RemoveFromStringBitmap(ctx context.Context, name string, v k := nameValue[string]{name: name, value: value} bitmap, ok := c.stringBitmaps[k] if !ok { - bitmap, err = c.st.ReconstructLatestStringBitmap(ctx, name, value) + bitmap, err = c.loadStringBitmap(ctx, k) if err != nil { - return fmt.Errorf("failed to reconstruct string bitmap %q=%q: %w", name, value, err) + return err } - c.stringOldBitmaps[k] = cloneBitmap(bitmap) - c.stringBitmaps[k] = bitmap } bitmap.Remove(id) @@ -80,12 +121,10 @@ func (c *bitmapCache) AddToNumericBitmap(ctx context.Context, name string, value k := nameValue[uint64]{name: name, value: value} bitmap, ok := c.numericBitmaps[k] if !ok { - bitmap, err = c.st.ReconstructLatestNumericBitmap(ctx, name, value) + bitmap, err = c.loadNumericBitmap(ctx, k) if err != nil { - return fmt.Errorf("failed to reconstruct numeric bitmap %q=%d: %w", name, value, err) + return err } - c.numericOldBitmaps[k] = cloneBitmap(bitmap) - c.numericBitmaps[k] = bitmap } bitmap.Add(id) @@ -96,18 +135,22 @@ func (c *bitmapCache) RemoveFromNumericBitmap(ctx context.Context, name string, k := nameValue[uint64]{name: name, value: value} bitmap, ok := c.numericBitmaps[k] if !ok { - bitmap, err = c.st.ReconstructLatestNumericBitmap(ctx, name, value) + bitmap, err = c.loadNumericBitmap(ctx, k) if err != nil { - return fmt.Errorf("failed to reconstruct numeric bitmap %q=%d: %w", name, value, err) + return err } - c.numericOldBitmaps[k] = cloneBitmap(bitmap) - c.numericBitmaps[k] = bitmap } bitmap.Remove(id) return nil } +// SetBlock updates the block number for the next round of changes. +// Call this between blocks to reuse the cache without re-reading from the DB. +func (c *bitmapCache) SetBlock(block uint64) { + c.block = block +} + func (c *bitmapCache) Flush(ctx context.Context) (err error) { // RunOptimize all bitmaps in parallel (CPU-bound). eg := &errgroup.Group{} @@ -143,19 +186,12 @@ func (c *bitmapCache) Flush(ctx context.Context) (err error) { oldBitmap = store.NewBitmap() } - // Skip if bitmap hasn't changed. + // Skip if bitmap hasn't changed since last flush. if newBitmap.Bitmap.Equals(oldBitmap.Bitmap) { continue } - deltaCount, err := c.st.CountDeltasSinceLastKeyframeString(ctx, store.CountDeltasSinceLastKeyframeStringParams{ - Name: k.name, Value: k.value, - }) - if err != nil { - return fmt.Errorf("failed to count deltas for string %q=%q: %w", k.name, k.value, err) - } - - isKeyframe := deltaCount >= store.KeyframeInterval || oldBitmap.IsEmpty() + isKeyframe := c.stringDeltaCounts[k] >= store.KeyframeInterval || oldBitmap.IsEmpty() if isKeyframe { err = c.st.InsertStringBitmapEntry(ctx, store.InsertStringBitmapEntryParams{ @@ -165,6 +201,7 @@ func (c *bitmapCache) Flush(ctx context.Context) (err error) { IsFullBitmap: true, Bitmap: newBitmap, }) + c.stringDeltaCounts[k] = 0 } else { delta := store.ComputeDelta(oldBitmap.Bitmap, newBitmap.Bitmap) err = c.st.InsertStringBitmapEntry(ctx, store.InsertStringBitmapEntryParams{ @@ -174,10 +211,14 @@ func (c *bitmapCache) Flush(ctx context.Context) (err error) { IsFullBitmap: false, Bitmap: delta, }) + c.stringDeltaCounts[k]++ } if err != nil { return fmt.Errorf("failed to insert string bitmap entry %q=%q: %w", k.name, k.value, err) } + + // Promote only changed bitmaps. + c.stringOldBitmaps[k] = cloneBitmap(newBitmap) } // Write numeric bitmaps as keyframes or deltas. @@ -191,14 +232,7 @@ func (c *bitmapCache) Flush(ctx context.Context) (err error) { continue } - deltaCount, err := c.st.CountDeltasSinceLastKeyframeNumeric(ctx, store.CountDeltasSinceLastKeyframeNumericParams{ - Name: k.name, Value: k.value, - }) - if err != nil { - return fmt.Errorf("failed to count deltas for numeric %q=%d: %w", k.name, k.value, err) - } - - isKeyframe := deltaCount >= store.KeyframeInterval || oldBitmap.IsEmpty() + isKeyframe := c.numericDeltaCounts[k] >= store.KeyframeInterval || oldBitmap.IsEmpty() if isKeyframe { err = c.st.InsertNumericBitmapEntry(ctx, store.InsertNumericBitmapEntryParams{ @@ -208,6 +242,7 @@ func (c *bitmapCache) Flush(ctx context.Context) (err error) { IsFullBitmap: true, Bitmap: newBitmap, }) + c.numericDeltaCounts[k] = 0 } else { delta := store.ComputeDelta(oldBitmap.Bitmap, newBitmap.Bitmap) err = c.st.InsertNumericBitmapEntry(ctx, store.InsertNumericBitmapEntryParams{ @@ -217,10 +252,13 @@ func (c *bitmapCache) Flush(ctx context.Context) (err error) { IsFullBitmap: false, Bitmap: delta, }) + c.numericDeltaCounts[k]++ } if err != nil { return fmt.Errorf("failed to insert numeric bitmap entry %q=%d: %w", k.name, k.value, err) } + + c.numericOldBitmaps[k] = cloneBitmap(newBitmap) } return nil diff --git a/sqlitestore.go b/sqlitestore.go index dee8771..4e85ab8 100644 --- a/sqlitestore.go +++ b/sqlitestore.go @@ -160,6 +160,9 @@ func (s *SQLiteStore) FollowEvents(ctx context.Context, iterator arkivevents.Bat metricOperationStarted.Inc(1) + // Create bitmap cache once per batch; reuse across blocks. + cache := newBitmapCache(st, firstBlock) + mainLoop: for _, block := range batch.Batch.Blocks { @@ -173,8 +176,7 @@ func (s *SQLiteStore) FollowEvents(ctx context.Context, iterator arkivevents.Bat } blockStat := stats[block.Number] - // Per-block bitmap cache for bitemporality. - cache := newBitmapCache(st, block.Number) + cache.SetBlock(block.Number) updatesMap := map[common.Hash][]*events.OPUpdate{} From c458284d8ad2440a0da3681a52781ab3d9700201 Mon Sep 17 00:00:00 2001 From: Dragan Milic Date: Wed, 11 Mar 2026 13:26:31 +0100 Subject: [PATCH 5/5] moved flush at the proper place, still no effect --- bitmap_cache.go | 22 ++-------------------- sqlitestore.go | 10 +++++----- 2 files changed, 7 insertions(+), 25 deletions(-) diff --git a/bitmap_cache.go b/bitmap_cache.go index 1fff047..4a103af 100644 --- a/bitmap_cache.go +++ b/bitmap_cache.go @@ -3,10 +3,8 @@ package sqlitebitmapstore import ( "context" "fmt" - "runtime" "github.com/Arkiv-Network/sqlite-bitmap-store/store" - "golang.org/x/sync/errgroup" ) type nameValue[T any] struct { @@ -152,33 +150,17 @@ func (c *bitmapCache) SetBlock(block uint64) { } func (c *bitmapCache) Flush(ctx context.Context) (err error) { - // RunOptimize all bitmaps in parallel (CPU-bound). - eg := &errgroup.Group{} - eg.SetLimit(runtime.NumCPU()) - for _, bitmap := range c.stringBitmaps { if !bitmap.IsEmpty() { - eg.Go(func() error { - bitmap.RunOptimize() - return nil - }) + bitmap.RunOptimize() } } - for _, bitmap := range c.numericBitmaps { if !bitmap.IsEmpty() { - eg.Go(func() error { - bitmap.RunOptimize() - return nil - }) + bitmap.RunOptimize() } } - err = eg.Wait() - if err != nil { - return fmt.Errorf("failed to run optimize: %w", err) - } - // Write string bitmaps as keyframes or deltas. for k, newBitmap := range c.stringBitmaps { oldBitmap := c.stringOldBitmaps[k] diff --git a/sqlitestore.go b/sqlitestore.go index 4e85ab8..ad4a1eb 100644 --- a/sqlitestore.go +++ b/sqlitestore.go @@ -488,14 +488,14 @@ func (s *SQLiteStore) FollowEvents(ctx context.Context, iterator arkivevents.Bat } - err = cache.Flush(ctx) - if err != nil { - return fmt.Errorf("failed to flush bitmap cache for block %d: %w", block.Number, err) - } - s.log.Info("block updated", "block", block.Number, "creates", blockStat.creates, "updates", blockStat.updates, "deletes", blockStat.deletes, "extends", blockStat.extends, "ownerChanges", blockStat.ownerChanges) } + err = cache.Flush(ctx) + if err != nil { + return fmt.Errorf("failed to flush bitmap cache: %w", err) + } + err = st.UpsertLastBlock(ctx, lastBlock) if err != nil { return fmt.Errorf("failed to upsert last block: %w", err)