diff --git a/cli/cli/package.json b/cli/cli/package.json index 21299a4..803e10c 100644 --- a/cli/cli/package.json +++ b/cli/cli/package.json @@ -18,6 +18,7 @@ "@_all_docs/frame": "workspace:*", "@_all_docs/packument": "workspace:*", "@_all_docs/partition": "workspace:*", + "@_all_docs/view": "workspace:*", "@_all_docs/worker": "workspace:*", "@vltpkg/error-cause": "0.0.0-9", "debug": "^4.4.0", diff --git a/cli/cli/src/cmd/view/define.js b/cli/cli/src/cmd/view/define.js new file mode 100644 index 0000000..9ed8d8f --- /dev/null +++ b/cli/cli/src/cmd/view/define.js @@ -0,0 +1,99 @@ +import { View, ViewStore } from '@_all_docs/view'; +import { encodeOrigin } from '@_all_docs/cache'; + +export const usage = `Usage: _all_docs view define [options] + +Define a named view over cached registry data. + +A view is a predicate (origin filter) plus a projection (field selection). +Views are stored as JSON files and can be queried or joined. + +Options: + --origin Origin key (e.g., npm, paces.exale.com~javpt) + --registry Registry URL (converted to origin key internally) + --type Entity type: packument (default) or partition + --select Field selection expression + +Select Expression Syntax: + Simple fields: name, version, description + Nested fields: time.modified, repository.url + With transforms: versions|keys, dependencies|length + With aliases: versions|keys as version_list + +Available Transforms: + keys, values Object to array + length Array/string length + first, last First/last element + sort, reverse Sort/reverse array + unique, compact Dedupe/remove nulls + flatten Flatten nested arrays + +Examples: + _all_docs view define npm-packages --origin npm + _all_docs view define npm-versions --origin npm --select 'name, versions|keys as versions, time' + _all_docs view define private --registry https://npm.company.com --select 'name, versions|keys' +`; + +export const command = async (cli) => { + if (cli.values.help) { + console.log(usage); + return; + } + + const name = cli._[0]; + if (!name) { + console.error('Error: View name required'); + console.error('Usage: _all_docs view define --origin [--select ]'); + process.exit(1); + } + + // Validate name + if (!/^[a-zA-Z][a-zA-Z0-9_-]*$/.test(name)) { + console.error('Error: View name must start with a letter and contain only letters, numbers, underscores, and hyphens'); + process.exit(1); + } + + // Prioritize --registry if specified, otherwise use --origin + // Note: cli.values.origin has a default from jack.js, so we check registry first + let origin; + if (cli.values.registry) { + origin = encodeOrigin(cli.values.registry); + } else if (cli.values.origin && cli.values.origin !== 'https://replicate.npmjs.com') { + // User explicitly set --origin (not the default) + origin = cli.values.origin; + } else { + console.error('Error: --origin or --registry required'); + console.error('Example: _all_docs view define my-view --origin npm'); + process.exit(1); + } + + const view = new View({ + name, + origin, + registry: cli.values.registry || null, + type: cli.values.type || 'packument', + select: cli.values.select || null + }); + + const store = new ViewStore(cli.dir('config')); + + // Check if view already exists + if (await store.exists(name)) { + if (!cli.values.force) { + console.error(`Error: View '${name}' already exists. Use --force to overwrite.`); + process.exit(1); + } + } + + await store.save(view); + + console.log(`View '${name}' defined:`); + console.log(` Origin: ${origin}`); + if (cli.values.registry) { + console.log(` Registry: ${cli.values.registry}`); + } + console.log(` Type: ${view.type}`); + if (view.select) { + console.log(` Select: ${view.select}`); + } +}; diff --git a/cli/cli/src/cmd/view/delete.js b/cli/cli/src/cmd/view/delete.js new file mode 100644 index 0000000..5869278 --- /dev/null +++ b/cli/cli/src/cmd/view/delete.js @@ -0,0 +1,37 @@ +import { ViewStore } from '@_all_docs/view'; + +export const usage = `Usage: _all_docs view delete + +Delete a defined view. + +Options: + --force Skip confirmation + +Examples: + _all_docs view delete old-view + _all_docs view delete old-view --force +`; + +export const command = async (cli) => { + if (cli.values.help) { + console.log(usage); + return; + } + + const name = cli._[0]; + if (!name) { + console.error('Error: View name required'); + console.error('Usage: _all_docs view delete '); + process.exit(1); + } + + const store = new ViewStore(cli.dir('config')); + + if (!await store.exists(name)) { + console.error(`Error: View '${name}' does not exist`); + process.exit(1); + } + + await store.delete(name); + console.log(`View '${name}' deleted`); +}; diff --git a/cli/cli/src/cmd/view/index.js b/cli/cli/src/cmd/view/index.js new file mode 100644 index 0000000..c357733 --- /dev/null +++ b/cli/cli/src/cmd/view/index.js @@ -0,0 +1,32 @@ +/** + * View commands - define, query, and join views over cached data + */ +export { command as define, usage as defineUsage } from './define.js'; +export { command as list, usage as listUsage } from './list.js'; +export { command as show, usage as showUsage } from './show.js'; +export { command as deleteView, usage as deleteUsage } from './delete.js'; +export { command as query, usage as queryUsage } from './query.js'; +export { command as join, usage as joinUsage } from './join.js'; + +export const usage = `Usage: _all_docs view [options] + +Manage named views over cached registry data. + +Commands: + define Define a new view + list List all defined views + show Show view details + delete Delete a view + query Query a view (output ndjson) + join Join two views + +A view is a predicate (origin filter) plus a projection (field selection). +Views enable efficient queries and joins across different registry caches. + +Examples: + _all_docs view define npm-pkgs --origin npm + _all_docs view define npm-vers --origin npm --select 'name, versions|keys' + _all_docs view list + _all_docs view query npm-vers --limit 100 + _all_docs view join npm-vers cgr-vers --diff +`; diff --git a/cli/cli/src/cmd/view/join.js b/cli/cli/src/cmd/view/join.js new file mode 100644 index 0000000..7b0278c --- /dev/null +++ b/cli/cli/src/cmd/view/join.js @@ -0,0 +1,85 @@ +import { ViewStore, joinViews, diffViews } from '@_all_docs/view'; +import { Cache, createStorageDriver } from '@_all_docs/cache'; + +export const usage = `Usage: _all_docs view join [options] + +Join two views on their common key (package name). + +Join Types: + --left Include all from left, matching from right (default) + --inner Only include records present in both views + --right Include all from right, matching from left + --full Include all records from both views + --diff Output records in left but not in right + +Options: + --on Join key field (default: name) + --limit Maximum records to return + --json Output as ndjson (default) + +Examples: + _all_docs view join npm-packages cgr-packages + _all_docs view join npm-packages cgr-packages --inner + _all_docs view join npm-packages cgr-packages --diff + _all_docs view join npm-versions cgr-versions --limit 1000 +`; + +export const command = async (cli) => { + if (cli.values.help) { + console.log(usage); + return; + } + + const leftName = cli._[0]; + const rightName = cli._[1]; + + if (!leftName || !rightName) { + console.error('Error: Two view names required'); + console.error('Usage: _all_docs view join '); + process.exit(1); + } + + const store = new ViewStore(cli.dir('config')); + + let leftView, rightView; + try { + leftView = await store.load(leftName); + rightView = await store.load(rightName); + } catch (err) { + console.error(`Error: ${err.message}`); + process.exit(1); + } + + const driver = await createStorageDriver({ CACHE_DIR: cli.dir('packuments') }); + const cache = new Cache({ path: cli.dir('packuments'), driver }); + + // Determine join type + let type = 'left'; + if (cli.values.inner) type = 'inner'; + else if (cli.values.right) type = 'right'; + else if (cli.values.full) type = 'full'; + + const options = { + type, + on: cli.values.on || 'name', + limit: cli.values.limit ? parseInt(cli.values.limit, 10) : undefined + }; + + try { + // Special case for diff + if (cli.values.diff) { + for await (const record of diffViews(leftView, rightView, cache, options)) { + console.log(JSON.stringify(record)); + } + return; + } + + // Regular join + for await (const record of joinViews(leftView, rightView, cache, options)) { + console.log(JSON.stringify(record)); + } + } catch (err) { + console.error(`Error joining views: ${err.message}`); + process.exit(1); + } +}; diff --git a/cli/cli/src/cmd/view/list.js b/cli/cli/src/cmd/view/list.js new file mode 100644 index 0000000..dd18145 --- /dev/null +++ b/cli/cli/src/cmd/view/list.js @@ -0,0 +1,59 @@ +import { ViewStore } from '@_all_docs/view'; + +export const usage = `Usage: _all_docs view list + +List all defined views. + +Options: + --json Output as JSON array + +Examples: + _all_docs view list + _all_docs view list --json +`; + +export const command = async (cli) => { + if (cli.values.help) { + console.log(usage); + return; + } + + const store = new ViewStore(cli.dir('config')); + const names = await store.list(); + + if (names.length === 0) { + console.log('No views defined.'); + console.log(''); + console.log('Create a view with:'); + console.log(' _all_docs view define --origin [--select ]'); + return; + } + + if (cli.values.json) { + const views = []; + for (const name of names) { + const view = await store.load(name); + views.push(view.toJSON()); + } + console.log(JSON.stringify(views, null, 2)); + return; + } + + console.log('Defined views:'); + console.log(''); + + for (const name of names) { + try { + const view = await store.load(name); + console.log(` ${name}`); + console.log(` Origin: ${view.origin}`); + console.log(` Type: ${view.type}`); + if (view.select) { + console.log(` Select: ${view.select}`); + } + console.log(''); + } catch (err) { + console.log(` ${name} (error loading: ${err.message})`); + } + } +}; diff --git a/cli/cli/src/cmd/view/query.js b/cli/cli/src/cmd/view/query.js new file mode 100644 index 0000000..583ff91 --- /dev/null +++ b/cli/cli/src/cmd/view/query.js @@ -0,0 +1,75 @@ +import { ViewStore, queryView, countView, collectView } from '@_all_docs/view'; +import { Cache, createStorageDriver } from '@_all_docs/cache'; + +export const usage = `Usage: _all_docs view query [options] + +Query a defined view, outputting matching records. + +Options: + --limit Maximum records to return + --count Only output the count of matching records + --collect Collect all results into a JSON array + --filter Filter expression (e.g., "name=lodash", "versions|length>10") + --json Output as ndjson (default) + +Examples: + _all_docs view query npm-packages + _all_docs view query npm-versions --limit 100 + _all_docs view query npm-packages --count + _all_docs view query npm-packages --filter "name=lodash" + _all_docs view query npm-versions --collect > all-versions.json +`; + +export const command = async (cli) => { + if (cli.values.help) { + console.log(usage); + return; + } + + const name = cli._[0]; + if (!name) { + console.error('Error: View name required'); + console.error('Usage: _all_docs view query '); + process.exit(1); + } + + const store = new ViewStore(cli.dir('config')); + + let view; + try { + view = await store.load(name); + } catch (err) { + console.error(`Error: ${err.message}`); + process.exit(1); + } + + const driver = await createStorageDriver({ CACHE_DIR: cli.dir('packuments') }); + const cache = new Cache({ path: cli.dir('packuments'), driver }); + + const options = { + limit: cli.values.limit ? parseInt(cli.values.limit, 10) : undefined, + where: cli.values.filter || undefined + }; + + try { + if (cli.values.count) { + const count = await countView(view, cache, options); + console.log(count); + return; + } + + if (cli.values.collect) { + const results = await collectView(view, cache, options); + console.log(JSON.stringify(results, null, 2)); + return; + } + + // Stream ndjson output + for await (const record of queryView(view, cache, options)) { + console.log(JSON.stringify(record)); + } + } catch (err) { + console.error(`Error querying view: ${err.message}`); + process.exit(1); + } +}; diff --git a/cli/cli/src/cmd/view/show.js b/cli/cli/src/cmd/view/show.js new file mode 100644 index 0000000..b0a1a7a --- /dev/null +++ b/cli/cli/src/cmd/view/show.js @@ -0,0 +1,55 @@ +import { ViewStore } from '@_all_docs/view'; + +export const usage = `Usage: _all_docs view show + +Show details of a defined view. + +Options: + --json Output as JSON + +Examples: + _all_docs view show npm-versions + _all_docs view show npm-versions --json +`; + +export const command = async (cli) => { + if (cli.values.help) { + console.log(usage); + return; + } + + const name = cli._[0]; + if (!name) { + console.error('Error: View name required'); + console.error('Usage: _all_docs view show '); + process.exit(1); + } + + const store = new ViewStore(cli.dir('config')); + + try { + const view = await store.load(name); + + if (cli.values.json) { + console.log(JSON.stringify(view.toJSON(), null, 2)); + return; + } + + console.log(`View: ${view.name}`); + console.log(''); + console.log(` Origin: ${view.origin}`); + if (view.registry) { + console.log(` Registry: ${view.registry}`); + } + console.log(` Type: ${view.type}`); + if (view.select) { + console.log(` Select: ${view.select}`); + } + console.log(` Created: ${view.createdAt}`); + console.log(''); + console.log(`Cache key prefix: ${view.getCacheKeyPrefix()}`); + } catch (err) { + console.error(`Error: ${err.message}`); + process.exit(1); + } +}; diff --git a/cli/cli/src/jack.js b/cli/cli/src/jack.js index eb7e45c..728eae4 100644 --- a/cli/cli/src/jack.js +++ b/cli/cli/src/jack.js @@ -15,7 +15,8 @@ const fullCommands = { cache: 'cache', exec: 'exec', packument: 'packument', - partition: 'partition' + partition: 'partition', + view: 'view' }; const aliases = { @@ -25,7 +26,8 @@ const aliases = { ch: 'cache', ex: 'exec', pku: 'packument', - prt: 'partition' + prt: 'partition', + vw: 'view' }; const commands = { @@ -243,6 +245,65 @@ const cli = ack } }) + // View command options + .opt({ + type: { + hint: 'type', + description: `Entity type for view (packument or partition)` + }, + select: { + hint: 'expr', + description: `Field selection expression for view projection + + Syntax: field1, field2|transform as alias + Transforms: keys, values, length, first, last, sort, unique + ` + }, + filter: { + hint: 'expr', + description: `Filter expression for view query (e.g., "name=lodash")` + }, + on: { + hint: 'field', + description: `Join key field (default: name)` + }, + limit: { + hint: 'n', + description: `Maximum records to return` + } + }) + + .flag({ + force: { + short: 'f', + description: 'Force overwrite existing view definition' + }, + json: { + description: 'Output as JSON' + }, + count: { + description: 'Only output the count of matching records' + }, + collect: { + description: 'Collect all results into a JSON array' + }, + left: { + description: 'Left join - include all from left view (default join type)' + }, + inner: { + description: 'Inner join - only records present in both views' + }, + right: { + description: 'Right join - include all from right view' + }, + full: { + description: 'Full join - include all records from both views' + }, + diff: { + description: 'Set difference - records in left but not in right' + } + }) + .flag({ version: { short: 'v', diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 83d5cc0..0e32d3a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -86,6 +86,9 @@ importers: '@_all_docs/partition': specifier: workspace:* version: link:../../src/partition + '@_all_docs/view': + specifier: workspace:* + version: link:../../src/view '@_all_docs/worker': specifier: workspace:* version: link:../../workers/worker @@ -204,6 +207,12 @@ importers: src/types: {} + src/view: + dependencies: + '@_all_docs/cache': + specifier: workspace:* + version: link:../cache + workers/cloudflare: dependencies: '@_all_docs/types': diff --git a/src/cache/cache-key.js b/src/cache/cache-key.js index de3738d..420f10a 100644 --- a/src/cache/cache-key.js +++ b/src/cache/cache-key.js @@ -59,7 +59,7 @@ export function createPackumentKey(packageName, origin = 'https://registry.npmjs * @param {string} origin - Full origin URL * @returns {string} Short origin key */ -function encodeOrigin(origin) { +export function encodeOrigin(origin) { // Handle bare hostnames (no protocol) if (!origin.includes('://')) { origin = 'https://' + origin; diff --git a/src/cache/index.js b/src/cache/index.js index c53b703..5c7e66b 100644 --- a/src/cache/index.js +++ b/src/cache/index.js @@ -1,7 +1,7 @@ export { Cache } from './cache.js'; export { BaseHTTPClient, createAgent, createDispatcher } from './http.js'; export { CacheEntry } from './entry.js'; -export { createCacheKey, decodeCacheKey, createPartitionKey, createPackumentKey } from './cache-key.js'; +export { createCacheKey, decodeCacheKey, createPartitionKey, createPackumentKey, encodeOrigin } from './cache-key.js'; export { PartitionCheckpoint } from './checkpoint.js'; export { createStorageDriver } from './storage-driver.js'; export { AuthError, TempError, PermError, categorizeHttpError } from './errors.js'; \ No newline at end of file diff --git a/src/config/index.js b/src/config/index.js index 69f8683..f13c92a 100644 --- a/src/config/index.js +++ b/src/config/index.js @@ -10,6 +10,7 @@ export default class Config { this.dirs = { partitions: this.xdg.cache('partitions'), packuments: this.xdg.cache('packuments'), + config: this.xdg.config(), logs: this.xdg.data('logs'), sessions: this.xdg.data('sessions') }; diff --git a/src/view/README.md b/src/view/README.md new file mode 100644 index 0000000..432db2a --- /dev/null +++ b/src/view/README.md @@ -0,0 +1,248 @@ +# @_all_docs/view + +Views provide a way to query and join cached registry data using origin expressions. + +## What is a View? + +A **view** is: +1. A **predicate** (origin filter) that selects which cache entries to scan +2. A **projection** (field selection) that transforms each record + +Views are stored as JSON files and can be queried or joined with other views. + +## Why Views? + +The `_all_docs` cache stores packuments keyed by `v1:{type}:{origin}:{hex-name}`. When you want to find packages with `preinstall` scripts that don't declare `node-gyp` as a dependency (a potential supply chain red flag), you need to: + +1. Scan all npm entries for packages with install scripts +2. For each, check if it depends on node-gyp +3. Output packages that have scripts but no node-gyp + +Without views, this requires knowing the internal cache key structure. Views abstract this away: + +```bash +# Define packages with install scripts +_all_docs view define has-scripts --origin npm \ + --select 'name, scripts.preinstall, scripts.postinstall' + +# Define packages that use node-gyp +_all_docs view define uses-gyp --origin npm \ + --select 'name, dependencies, devDependencies' + +# Find packages with install scripts but NO node-gyp dependency +_all_docs view join has-scripts uses-gyp --diff | \ + jq 'select(.left.scripts.preinstall != null or .left.scripts.postinstall != null)' +``` + +Similarly, views make it easy to extract dependency graph edges for services like [deps.dev](https://deps.dev): + +```bash +# Define a view that extracts dependency edges +_all_docs view define npm-edges --origin npm \ + --select 'name, dependencies|keys as deps, devDependencies|keys as devDeps' + +# Stream edges in a format suitable for graph databases +_all_docs view query npm-edges | jq -c ' + .name as $src | + ((.deps // [])[] | {from: $src, to: ., type: "runtime"}), + ((.devDeps // [])[] | {from: $src, to: ., type: "dev"}) +' +``` + +## View Definition + +A view has these properties: + +| Property | Description | +|----------|-------------| +| `name` | Unique identifier for the view | +| `origin` | Encoded origin key (e.g., `npm`, `paces.exale.com~javpt`) | +| `registry` | Original registry URL (optional, for display) | +| `type` | Entity type: `packument` (default) or `partition` | +| `select` | Field projection expression | + +## Select Expression Syntax + +The select expression defines which fields to include and how to transform them. + +### Simple Fields +``` +name, description, license +``` + +### Nested Fields +``` +time.modified, repository.url +``` + +### Transforms +``` +versions|keys # Get object keys as array +versions|keys|length # Count of versions +dependencies|keys|sort # Sorted dependency names +``` + +### Aliases +``` +versions|keys as version_list # Rename the output field +time.modified as modified # Simplify nested field name +``` + +### Available Transforms + +| Transform | Description | +|-----------|-------------| +| `keys` | Object keys as array | +| `values` | Object values as array | +| `length` | Array or string length | +| `first` | First element | +| `last` | Last element | +| `sort` | Sort array | +| `reverse` | Reverse array | +| `unique` | Deduplicate array | +| `compact` | Remove null/undefined | +| `flatten` | Flatten nested arrays | +| `entries` | Object entries as [key, value] pairs | +| `sum` | Sum numeric array | +| `min` | Minimum value | +| `max` | Maximum value | + +## CLI Commands + +### Define a View +```bash +_all_docs view define --origin [--select ] +_all_docs view define --registry [--select ] + +# Examples +_all_docs view define npm-pkgs --origin npm +_all_docs view define npm-vers --origin npm --select 'name, versions|keys as versions' +_all_docs view define private --registry https://npm.company.com +_all_docs view define has-scripts --origin npm --select 'name, scripts.preinstall, scripts.postinstall' +``` + +### List Views +```bash +_all_docs view list +_all_docs view list --json +``` + +### Show View Details +```bash +_all_docs view show npm-vers +_all_docs view show npm-vers --json +``` + +### Query a View +```bash +_all_docs view query [options] + +# Examples +_all_docs view query npm-vers # Stream all as ndjson +_all_docs view query npm-vers --limit 100 # First 100 records +_all_docs view query npm-vers --count # Just the count +_all_docs view query npm-vers --filter "name=lodash" +_all_docs view query npm-vers --collect > all.json +``` + +### Join Two Views +```bash +_all_docs view join [options] + +# Join types +--left # All from left, matching from right (default) +--inner # Only records in both +--right # All from right, matching from left +--full # All records from both +--diff # Records in left but not in right +``` + +**Example: Find suspicious packages with install scripts but no node-gyp dependency** + +Packages with `preinstall` or `postinstall` scripts typically need them for native compilation (node-gyp). Packages with these scripts that *don't* depend on node-gyp may warrant investigation. + +```bash +# View 1: Packages with install scripts +_all_docs view define has-install-scripts --origin npm \ + --select 'name, scripts.preinstall, scripts.postinstall' \ + --where 'scripts.preinstall != null || scripts.postinstall != null' + +# View 2: Packages that depend on node-gyp +_all_docs view define uses-node-gyp --origin npm \ + --select 'name' \ + --where 'dependencies["node-gyp"] != null || devDependencies["node-gyp"] != null' + +# Find packages with install scripts but NO node-gyp dependency +_all_docs view join has-install-scripts uses-node-gyp --diff | head -100 +``` + +### Delete a View +```bash +_all_docs view delete +``` + +## Programmatic API + +```javascript +import { View, ViewStore, queryView, joinViews, diffViews } from '@_all_docs/view'; +import { Cache } from '@_all_docs/cache'; + +// Create and save a view +const view = new View({ + name: 'npm-packages', + origin: 'npm', + select: 'name, versions|keys as versions' +}); + +const store = new ViewStore('/path/to/config'); +await store.save(view); + +// Query a view +const cache = new Cache({ cacheDir: '/path/to/cache' }); + +for await (const record of queryView(view, cache)) { + console.log(record); // { name: 'lodash', versions: ['1.0.0', '2.0.0', ...] } +} + +// Extract dependency graph edges (deps.dev style) +const edgeView = new View({ + name: 'npm-edges', + origin: 'npm', + select: 'name, dependencies|keys as deps, devDependencies|keys as devDeps' +}); + +for await (const pkg of queryView(edgeView, cache)) { + for (const dep of pkg.deps || []) { + console.log({ from: pkg.name, to: dep, type: 'runtime' }); + } + for (const dep of pkg.devDeps || []) { + console.log({ from: pkg.name, to: dep, type: 'dev' }); + } +} +``` + +## How Joins Work + +Joins use **O(1) lookups** for the right side by constructing cache keys directly: + +1. Stream all records from the left view's origin prefix +2. For each record, construct the right-side cache key using the join key +3. Fetch the right record directly (no scanning) +4. Output based on join type + +This makes joins efficient even with millions of records. + +## Storage + +Views are stored in `{configDir}/views/{name}.view.json`: + +```json +{ + "name": "npm-versions", + "origin": "npm", + "registry": null, + "type": "packument", + "select": "name, versions|keys as versions, time.modified", + "createdAt": "2024-01-15T10:30:00.000Z" +} +``` diff --git a/src/view/index.js b/src/view/index.js new file mode 100644 index 0000000..c306711 --- /dev/null +++ b/src/view/index.js @@ -0,0 +1,13 @@ +/** + * View module - predicate + projection over cached data + */ +export { View } from './view.js'; +export { ViewStore } from './store.js'; +export { queryView, countView, collectView } from './query.js'; +export { joinViews, diffViews } from './join.js'; +export { + compileSelector, + compileFilter, + createProjection, + createFilter +} from './projection.js'; diff --git a/src/view/join.js b/src/view/join.js new file mode 100644 index 0000000..162b65d --- /dev/null +++ b/src/view/join.js @@ -0,0 +1,284 @@ +/** + * Join execution for views + */ +import { createProjection, createFilter } from './projection.js'; +import { queryView } from './query.js'; + +/** + * Create a cache key for a packument using a pre-encoded origin + * (The view.origin is already encoded, so we don't re-encode) + */ +function createPackumentKeyRaw(packageName, encodedOrigin) { + const nameHex = Array.from(new TextEncoder().encode(packageName)) + .map(b => b.toString(16).padStart(2, '0')) + .join(''); + return `v1:packument:${encodedOrigin}:${nameHex}`; +} + +/** + * Join two views on a common key + * @param {View} leftView - Left side of join + * @param {View} rightView - Right side of join + * @param {Cache} cache - The cache instance + * @param {Object} options - Join options + * @param {string} [options.on='name'] - Field to join on + * @param {string} [options.type='left'] - Join type: left, inner, right, full + * @param {string} [options.select] - Output projection + * @param {string} [options.where] - Post-join filter + * @param {boolean} [options.progress] - Show progress on stderr + * @yields {Object} Joined records + */ +export async function* joinViews(leftView, rightView, cache, options = {}) { + const { + on = 'name', + type = 'left', + select, + where, + progress = false + } = options; + + // Handle right join by swapping + if (type === 'right') { + yield* joinViews(rightView, leftView, cache, { + ...options, + type: 'left', + select: select?.replace(/\.left\b/g, '.__LEFT__') + .replace(/\.right\b/g, '.left') + .replace(/\.__LEFT__\b/g, '.right') + }); + return; + } + + // Handle full join + if (type === 'full') { + yield* fullJoin(leftView, rightView, cache, options); + return; + } + + // Left join or inner join + yield* leftOrInnerJoin(leftView, rightView, cache, options); +} + +/** + * Left or inner join implementation + */ +async function* leftOrInnerJoin(leftView, rightView, cache, options) { + const { + on = 'name', + type = 'left', + select, + where, + limit, + progress = false + } = options; + + const project = createProjection({ select }); + const filter = createFilter({ where }); + + // Compile left view's projection + const projectLeft = createProjection({ select: leftView.select }); + + // Compile right view's projection + const projectRight = createProjection({ select: rightView.select }); + + let count = 0; + let yielded = 0; + + // Stream the left view + for await (const key of cache.keys(leftView.getCacheKeyPrefix())) { + // Check limit + if (limit && yielded >= limit) break; + + count++; + + if (progress && count % 5000 === 0) { + process.stderr.write(`\rJoin: processed ${count}, yielded ${yielded}...`); + } + + try { + const leftEntry = await cache.fetch(key); + if (!leftEntry) continue; + + // Cache entries wrap the response - packument is in body + const leftValue = leftEntry.body || leftEntry; + const leftRecord = projectLeft(leftValue); + const joinKey = leftRecord[on]; + + if (!joinKey) continue; + + // Construct the right cache key directly - O(1) lookup! + const rightCacheKey = createPackumentKeyRaw(joinKey, rightView.origin); + + let rightRecord = null; + try { + const rightEntry = await cache.fetch(rightCacheKey); + if (rightEntry) { + const rightValue = rightEntry.body || rightEntry; + rightRecord = projectRight(rightValue); + } + } catch { + // Right record not found + } + + // Inner join: skip if no right match + if (type === 'inner' && !rightRecord) continue; + + // Build joined record + const joined = { + [on]: joinKey, + left: leftRecord, + right: rightRecord + }; + + // Apply post-join projection + const result = project(joined); + + // Apply post-join filter + if (!filter(result)) continue; + + yielded++; + yield result; + } catch (err) { + if (progress) { + process.stderr.write(`\nError in join: ${err.message}\n`); + } + } + } + + if (progress) { + process.stderr.write(`\rJoin: processed ${count}, yielded ${yielded} \n`); + } +} + +/** + * Full outer join implementation + */ +async function* fullJoin(leftView, rightView, cache, options) { + const { + on = 'name', + select, + where, + limit, + progress = false + } = options; + + const project = createProjection({ select }); + const filter = createFilter({ where }); + const projectLeft = createProjection({ select: leftView.select }); + const projectRight = createProjection({ select: rightView.select }); + + // Track which right keys we've seen + const seenRightKeys = new Set(); + + let count = 0; + let yielded = 0; + + // First pass: left join + for await (const key of cache.keys(leftView.getCacheKeyPrefix())) { + // Check limit + if (limit && yielded >= limit) break; + + count++; + + if (progress && count % 5000 === 0) { + process.stderr.write(`\rFull join pass 1: processed ${count}...`); + } + + try { + const leftEntry = await cache.fetch(key); + if (!leftEntry) continue; + + const leftValue = leftEntry.body || leftEntry; + const leftRecord = projectLeft(leftValue); + const joinKey = leftRecord[on]; + + if (!joinKey) continue; + + const rightCacheKey = createPackumentKeyRaw(joinKey, rightView.origin); + + let rightRecord = null; + try { + const rightEntry = await cache.fetch(rightCacheKey); + if (rightEntry) { + const rightValue = rightEntry.body || rightEntry; + rightRecord = projectRight(rightValue); + seenRightKeys.add(joinKey); + } + } catch { + // Right not found + } + + const joined = { + [on]: joinKey, + left: leftRecord, + right: rightRecord + }; + + const result = project(joined); + if (!filter(result)) continue; + + yielded++; + yield result; + } catch { + // Skip errors + } + } + + if (progress) { + process.stderr.write(`\rFull join pass 2: scanning right side... \n`); + } + + // Second pass: right-only records + let rightCount = 0; + for await (const key of cache.keys(rightView.getCacheKeyPrefix())) { + // Check limit + if (limit && yielded >= limit) break; + + rightCount++; + + if (progress && rightCount % 5000 === 0) { + process.stderr.write(`\rFull join pass 2: processed ${rightCount}...`); + } + + try { + const rightEntry = await cache.fetch(key); + if (!rightEntry) continue; + + const rightValue = rightEntry.body || rightEntry; + const rightRecord = projectRight(rightValue); + const joinKey = rightRecord[on]; + + if (!joinKey || seenRightKeys.has(joinKey)) continue; + + const joined = { + [on]: joinKey, + left: null, + right: rightRecord + }; + + const result = project(joined); + if (!filter(result)) continue; + + yielded++; + yield result; + } catch { + // Skip errors + } + } + + if (progress) { + process.stderr.write(`\rFull join complete: yielded ${yielded} \n`); + } +} + +/** + * Compute set difference between two views + * Returns records from left that don't exist in right + */ +export async function* diffViews(leftView, rightView, cache, options = {}) { + yield* joinViews(leftView, rightView, cache, { + ...options, + type: 'left', + where: options.where ? `${options.where} && right == null` : 'right == null' + }); +} diff --git a/src/view/package.json b/src/view/package.json new file mode 100644 index 0000000..7a21007 --- /dev/null +++ b/src/view/package.json @@ -0,0 +1,16 @@ +{ + "name": "@_all_docs/view", + "version": "1.0.0", + "description": "View abstraction for _all_docs cache queries", + "type": "module", + "main": "index.js", + "exports": { + ".": "./index.js" + }, + "scripts": { + "test": "node --test *.test.js" + }, + "dependencies": { + "@_all_docs/cache": "workspace:*" + } +} diff --git a/src/view/projection.js b/src/view/projection.js new file mode 100644 index 0000000..8e89f34 --- /dev/null +++ b/src/view/projection.js @@ -0,0 +1,199 @@ +/** + * Projection module - field selection and transformation + * + * Supports simple selector syntax: "name, versions|keys, time.modified as modified" + */ + +// Built-in transforms +const TRANSFORMS = { + keys: (obj) => obj ? Object.keys(obj) : [], + values: (obj) => obj ? Object.values(obj) : [], + length: (x) => x?.length ?? 0, + first: (arr) => arr?.[0], + last: (arr) => arr?.[arr.length - 1], + sort: (arr) => [...(arr || [])].sort(), + reverse: (arr) => [...(arr || [])].reverse(), + unique: (arr) => [...new Set(arr || [])], + flatten: (arr) => (arr || []).flat(), + compact: (arr) => (arr || []).filter(Boolean), + entries: (obj) => obj ? Object.entries(obj) : [], + sum: (arr) => (arr || []).reduce((a, b) => a + b, 0), + min: (arr) => arr?.length ? Math.min(...arr) : null, + max: (arr) => arr?.length ? Math.max(...arr) : null, +}; + +/** + * Get a nested value from an object using dot notation + * @param {Object} obj - Source object + * @param {string} path - Dot-separated path (e.g., "time.modified") + */ +function getPath(obj, path) { + if (!path || path === '.') return obj; + + const parts = path.split('.'); + let value = obj; + + for (const part of parts) { + if (value == null) return undefined; + value = value[part]; + } + + return value; +} + +/** + * Apply a transform function to a value + */ +function applyTransform(value, transformName) { + const fn = TRANSFORMS[transformName]; + if (!fn) { + throw new Error(`Unknown transform: ${transformName}. Available: ${Object.keys(TRANSFORMS).join(', ')}`); + } + return fn(value); +} + +/** + * Parse a field expression like "versions|keys|length as version_count" + * @returns {{ path: string, transforms: string[], alias: string }} + */ +function parseFieldExpr(expr) { + // Handle "expr as alias" syntax + let alias = null; + let mainExpr = expr.trim(); + + const asMatch = mainExpr.match(/^(.+)\s+as\s+(\w+)$/i); + if (asMatch) { + mainExpr = asMatch[1].trim(); + alias = asMatch[2]; + } + + // Parse path and transforms: "versions|keys|length" + const parts = mainExpr.split('|').map(p => p.trim()); + const path = parts[0]; + const transforms = parts.slice(1); + + // Default alias is the path (or last path segment) + if (!alias) { + alias = path.includes('.') ? path.split('.').pop() : path; + // If there are transforms, append them to alias + if (transforms.length > 0) { + alias = `${alias}_${transforms[transforms.length - 1]}`; + } + } + + return { path, transforms, alias }; +} + +/** + * Compile a simple selector expression into a projection function + * @param {string} selectExpr - e.g., "name, versions|keys as versions, time.modified" + * @returns {Function} Projection function + */ +export function compileSelector(selectExpr) { + if (!selectExpr) return (obj) => obj; + + // Parse the comma-separated field expressions + const fields = selectExpr.split(',').map(f => parseFieldExpr(f.trim())); + + return (obj) => { + const result = {}; + + for (const { path, transforms, alias } of fields) { + let value = getPath(obj, path); + + // Apply transforms in order + for (const t of transforms) { + value = applyTransform(value, t); + } + + result[alias] = value; + } + + return result; + }; +} + +/** + * Compile a filter expression + * Supports simple comparisons: "versions|length > 10" + */ +export function compileFilter(filterExpr) { + if (!filterExpr) return () => true; + + // Simple comparison patterns + const comparisonMatch = filterExpr.match(/^(.+?)\s*(==|!=|>=|<=|>|<)\s*(.+)$/); + + if (comparisonMatch) { + const [, leftExpr, op, rightExpr] = comparisonMatch; + const { path: leftPath, transforms: leftTransforms } = parseFieldExpr(leftExpr.trim()); + const rightValue = parseValue(rightExpr.trim()); + + return (obj) => { + let leftValue = getPath(obj, leftPath); + for (const t of leftTransforms) { + leftValue = applyTransform(leftValue, t); + } + + switch (op) { + case '==': return leftValue === rightValue; + case '!=': return leftValue !== rightValue; + case '>': return leftValue > rightValue; + case '<': return leftValue < rightValue; + case '>=': return leftValue >= rightValue; + case '<=': return leftValue <= rightValue; + default: return true; + } + }; + } + + // Existence check: just a path + const { path, transforms } = parseFieldExpr(filterExpr); + return (obj) => { + let value = getPath(obj, path); + for (const t of transforms) { + value = applyTransform(value, t); + } + return Boolean(value); + }; +} + +/** + * Parse a literal value from a filter expression + */ +function parseValue(str) { + // Number + if (/^-?\d+(\.\d+)?$/.test(str)) { + return parseFloat(str); + } + // Boolean + if (str === 'true') return true; + if (str === 'false') return false; + if (str === 'null') return null; + // String (quoted) + if ((str.startsWith('"') && str.endsWith('"')) || + (str.startsWith("'") && str.endsWith("'"))) { + return str.slice(1, -1); + } + // Unquoted string + return str; +} + +/** + * Create a projection function from options + */ +export function createProjection(options = {}) { + if (options.select) { + return compileSelector(options.select); + } + return (obj) => obj; +} + +/** + * Create a filter function from options + */ +export function createFilter(options = {}) { + if (options.where) { + return compileFilter(options.where); + } + return () => true; +} diff --git a/src/view/projection.test.js b/src/view/projection.test.js new file mode 100644 index 0000000..b35fab3 --- /dev/null +++ b/src/view/projection.test.js @@ -0,0 +1,201 @@ +import { test, describe } from 'node:test'; +import assert from 'node:assert'; +import { + compileSelector, + compileFilter, + createProjection, + createFilter +} from './projection.js'; + +describe('compileSelector', () => { + test('returns identity function for null/undefined', () => { + const fn = compileSelector(null); + const obj = { name: 'test', value: 42 }; + assert.deepStrictEqual(fn(obj), obj); + }); + + test('selects simple fields', () => { + const fn = compileSelector('name, version'); + const result = fn({ name: 'lodash', version: '4.17.21', extra: 'ignored' }); + + assert.deepStrictEqual(result, { name: 'lodash', version: '4.17.21' }); + }); + + test('selects nested fields', () => { + const fn = compileSelector('name, time.modified'); + const result = fn({ + name: 'lodash', + time: { created: '2020-01-01', modified: '2024-01-01' } + }); + + assert.deepStrictEqual(result, { name: 'lodash', modified: '2024-01-01' }); + }); + + test('applies transforms', () => { + const fn = compileSelector('versions|keys'); + const result = fn({ + versions: { '1.0.0': {}, '2.0.0': {}, '3.0.0': {} } + }); + + assert.deepStrictEqual(result, { versions_keys: ['1.0.0', '2.0.0', '3.0.0'] }); + }); + + test('applies multiple transforms', () => { + const fn = compileSelector('versions|keys|length'); + const result = fn({ + versions: { '1.0.0': {}, '2.0.0': {}, '3.0.0': {} } + }); + + assert.deepStrictEqual(result, { versions_length: 3 }); + }); + + test('uses aliases', () => { + const fn = compileSelector('versions|keys as version_list'); + const result = fn({ + versions: { '1.0.0': {}, '2.0.0': {} } + }); + + assert.deepStrictEqual(result, { version_list: ['1.0.0', '2.0.0'] }); + }); + + test('handles null values gracefully', () => { + const fn = compileSelector('name, missing|keys'); + const result = fn({ name: 'test' }); + + assert.deepStrictEqual(result, { name: 'test', missing_keys: [] }); + }); +}); + +describe('transforms', () => { + test('keys transform', () => { + const fn = compileSelector('data|keys'); + assert.deepStrictEqual(fn({ data: { a: 1, b: 2 } }), { data_keys: ['a', 'b'] }); + }); + + test('values transform', () => { + const fn = compileSelector('data|values'); + assert.deepStrictEqual(fn({ data: { a: 1, b: 2 } }), { data_values: [1, 2] }); + }); + + test('length transform', () => { + const fn = compileSelector('items|length'); + assert.deepStrictEqual(fn({ items: [1, 2, 3] }), { items_length: 3 }); + }); + + test('first transform', () => { + const fn = compileSelector('items|first'); + assert.deepStrictEqual(fn({ items: ['a', 'b', 'c'] }), { items_first: 'a' }); + }); + + test('last transform', () => { + const fn = compileSelector('items|last'); + assert.deepStrictEqual(fn({ items: ['a', 'b', 'c'] }), { items_last: 'c' }); + }); + + test('sort transform', () => { + const fn = compileSelector('items|sort'); + assert.deepStrictEqual(fn({ items: ['c', 'a', 'b'] }), { items_sort: ['a', 'b', 'c'] }); + }); + + test('unique transform', () => { + const fn = compileSelector('items|unique'); + assert.deepStrictEqual(fn({ items: [1, 2, 2, 3, 3, 3] }), { items_unique: [1, 2, 3] }); + }); + + test('compact transform', () => { + const fn = compileSelector('items|compact'); + assert.deepStrictEqual(fn({ items: [1, null, 2, undefined, 3] }), { items_compact: [1, 2, 3] }); + }); + + test('flatten transform', () => { + const fn = compileSelector('items|flatten'); + assert.deepStrictEqual(fn({ items: [[1, 2], [3, 4]] }), { items_flatten: [1, 2, 3, 4] }); + }); + + test('sum transform', () => { + const fn = compileSelector('items|sum'); + assert.deepStrictEqual(fn({ items: [1, 2, 3, 4] }), { items_sum: 10 }); + }); +}); + +describe('compileFilter', () => { + test('returns always-true for null/undefined', () => { + const fn = compileFilter(null); + assert.strictEqual(fn({ any: 'value' }), true); + }); + + test('filters with equality', () => { + const fn = compileFilter('name == lodash'); + assert.strictEqual(fn({ name: 'lodash' }), true); + assert.strictEqual(fn({ name: 'express' }), false); + }); + + test('filters with inequality', () => { + const fn = compileFilter('name != lodash'); + assert.strictEqual(fn({ name: 'lodash' }), false); + assert.strictEqual(fn({ name: 'express' }), true); + }); + + test('filters with greater than', () => { + const fn = compileFilter('count > 10'); + assert.strictEqual(fn({ count: 15 }), true); + assert.strictEqual(fn({ count: 5 }), false); + }); + + test('filters with less than', () => { + const fn = compileFilter('count < 10'); + assert.strictEqual(fn({ count: 5 }), true); + assert.strictEqual(fn({ count: 15 }), false); + }); + + test('filters with transforms', () => { + const fn = compileFilter('versions|keys|length > 5'); + assert.strictEqual(fn({ versions: { '1': {}, '2': {}, '3': {}, '4': {}, '5': {}, '6': {} } }), true); + assert.strictEqual(fn({ versions: { '1': {}, '2': {} } }), false); + }); + + test('filters with null comparison', () => { + const fn = compileFilter('value == null'); + assert.strictEqual(fn({ value: null }), true); + assert.strictEqual(fn({ value: 'something' }), false); + }); + + test('filters with boolean comparison', () => { + const fn = compileFilter('active == true'); + assert.strictEqual(fn({ active: true }), true); + assert.strictEqual(fn({ active: false }), false); + }); + + test('existence filter', () => { + const fn = compileFilter('name'); + assert.strictEqual(fn({ name: 'lodash' }), true); + assert.strictEqual(fn({ name: '' }), false); + assert.strictEqual(fn({ name: null }), false); + }); +}); + +describe('createProjection', () => { + test('returns identity when no select', () => { + const fn = createProjection({}); + const obj = { a: 1, b: 2 }; + assert.deepStrictEqual(fn(obj), obj); + }); + + test('uses select option', () => { + const fn = createProjection({ select: 'name' }); + assert.deepStrictEqual(fn({ name: 'test', extra: 'ignored' }), { name: 'test' }); + }); +}); + +describe('createFilter', () => { + test('returns always-true when no where', () => { + const fn = createFilter({}); + assert.strictEqual(fn({ any: 'value' }), true); + }); + + test('uses where option', () => { + const fn = createFilter({ where: 'count > 0' }); + assert.strictEqual(fn({ count: 5 }), true); + assert.strictEqual(fn({ count: 0 }), false); + }); +}); diff --git a/src/view/query.js b/src/view/query.js new file mode 100644 index 0000000..441b253 --- /dev/null +++ b/src/view/query.js @@ -0,0 +1,90 @@ +/** + * Query execution for views + */ +import { createProjection, createFilter } from './projection.js'; + +/** + * Query a view, yielding projected records + * @param {View} view - The view to query + * @param {Cache} cache - The cache instance + * @param {Object} options - Query options + * @param {number} [options.limit] - Maximum records to return + * @param {string} [options.where] - Additional filter expression + * @param {boolean} [options.progress] - Show progress on stderr + * @yields {Object} Projected records + */ +export async function* queryView(view, cache, options = {}) { + const { limit, where, progress = false } = options; + + // Compile projection from view's select + const project = createProjection({ select: view.select }); + + // Compile additional filter if provided + const filter = createFilter({ where }); + + const prefix = view.getCacheKeyPrefix(); + let count = 0; + let yielded = 0; + + for await (const key of cache.keys(prefix)) { + // Check limit + if (limit && yielded >= limit) break; + + count++; + + // Progress reporting + if (progress && count % 10000 === 0) { + process.stderr.write(`\rProcessed ${count} records, yielded ${yielded}...`); + } + + try { + const entry = await cache.fetch(key); + if (!entry) continue; + + // Cache entries wrap the response - packument is in body + const value = entry.body || entry; + + // Apply view's projection + const projected = project(value); + + // Apply additional filter + if (!filter(projected)) continue; + + yielded++; + yield projected; + } catch (err) { + // Log and continue on individual record errors + if (progress) { + process.stderr.write(`\nError processing ${key}: ${err.message}\n`); + } + } + } + + // Clear progress line + if (progress && count > 0) { + process.stderr.write(`\rProcessed ${count} records, yielded ${yielded} \n`); + } +} + +/** + * Count records in a view (without yielding them) + */ +export async function countView(view, cache, options = {}) { + let count = 0; + for await (const _ of queryView(view, cache, options)) { + count++; + } + return count; +} + +/** + * Collect all records from a view into an array + * Use with caution for large views! + */ +export async function collectView(view, cache, options = {}) { + const results = []; + for await (const record of queryView(view, cache, options)) { + results.push(record); + } + return results; +} diff --git a/src/view/store.js b/src/view/store.js new file mode 100644 index 0000000..1fdd3e1 --- /dev/null +++ b/src/view/store.js @@ -0,0 +1,75 @@ +/** + * ViewStore - file-based storage for view definitions + */ +import { readFile, writeFile, mkdir, readdir, unlink } from 'node:fs/promises'; +import { join } from 'node:path'; +import { View } from './view.js'; + +export class ViewStore { + constructor(baseDir) { + this.viewsDir = join(baseDir, 'views'); + } + + async init() { + await mkdir(this.viewsDir, { recursive: true }); + } + + viewPath(name) { + // Sanitize name to prevent directory traversal + const safeName = name.replace(/[^a-zA-Z0-9_-]/g, '_'); + return join(this.viewsDir, `${safeName}.view.json`); + } + + async save(view) { + await this.init(); + const json = JSON.stringify(view.toJSON(), null, 2); + await writeFile(this.viewPath(view.name), json, 'utf8'); + } + + async load(name) { + try { + const data = await readFile(this.viewPath(name), 'utf8'); + return View.fromJSON(JSON.parse(data)); + } catch (err) { + if (err.code === 'ENOENT') { + throw new Error(`View '${name}' not found. Run '_all_docs view list' to see available views.`); + } + throw err; + } + } + + async list() { + try { + await this.init(); + const files = await readdir(this.viewsDir); + return files + .filter(f => f.endsWith('.view.json')) + .map(f => f.replace('.view.json', '')); + } catch (err) { + if (err.code === 'ENOENT') { + return []; + } + throw err; + } + } + + async delete(name) { + try { + await unlink(this.viewPath(name)); + } catch (err) { + if (err.code === 'ENOENT') { + throw new Error(`View '${name}' not found.`); + } + throw err; + } + } + + async exists(name) { + try { + await readFile(this.viewPath(name), 'utf8'); + return true; + } catch { + return false; + } + } +} diff --git a/src/view/store.test.js b/src/view/store.test.js new file mode 100644 index 0000000..9ab8ec1 --- /dev/null +++ b/src/view/store.test.js @@ -0,0 +1,105 @@ +import { test, describe, beforeEach, afterEach } from 'node:test'; +import assert from 'node:assert'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { ViewStore } from './store.js'; +import { View } from './view.js'; + +describe('ViewStore', () => { + let tempDir; + let store; + + beforeEach(async () => { + tempDir = await mkdtemp(join(tmpdir(), 'view-store-test-')); + store = new ViewStore(tempDir); + }); + + afterEach(async () => { + await rm(tempDir, { recursive: true, force: true }); + }); + + test('saves and loads a view', async () => { + const view = new View({ + name: 'test-view', + origin: 'npm', + select: 'name, versions|keys' + }); + + await store.save(view); + const loaded = await store.load('test-view'); + + assert.strictEqual(loaded.name, 'test-view'); + assert.strictEqual(loaded.origin, 'npm'); + assert.strictEqual(loaded.select, 'name, versions|keys'); + }); + + test('lists views', async () => { + const view1 = new View({ name: 'view-a', origin: 'npm' }); + const view2 = new View({ name: 'view-b', origin: 'private' }); + + await store.save(view1); + await store.save(view2); + + const names = await store.list(); + assert.deepStrictEqual(names.sort(), ['view-a', 'view-b']); + }); + + test('returns empty list when no views', async () => { + const names = await store.list(); + assert.deepStrictEqual(names, []); + }); + + test('checks if view exists', async () => { + const view = new View({ name: 'existing', origin: 'npm' }); + await store.save(view); + + assert.strictEqual(await store.exists('existing'), true); + assert.strictEqual(await store.exists('non-existing'), false); + }); + + test('deletes a view', async () => { + const view = new View({ name: 'to-delete', origin: 'npm' }); + await store.save(view); + + assert.strictEqual(await store.exists('to-delete'), true); + + await store.delete('to-delete'); + + assert.strictEqual(await store.exists('to-delete'), false); + }); + + test('throws when loading non-existent view', async () => { + await assert.rejects( + () => store.load('non-existent'), + /View 'non-existent' not found/ + ); + }); + + test('throws when deleting non-existent view', async () => { + await assert.rejects( + () => store.delete('non-existent'), + /View 'non-existent' not found/ + ); + }); + + test('sanitizes view names for storage', async () => { + const view = new View({ name: 'test_view-123', origin: 'npm' }); + await store.save(view); + + // Should be able to load it back + const loaded = await store.load('test_view-123'); + assert.strictEqual(loaded.name, 'test_view-123'); + }); + + test('overwrites existing view', async () => { + const view1 = new View({ name: 'overwrite-test', origin: 'npm' }); + await store.save(view1); + + const view2 = new View({ name: 'overwrite-test', origin: 'private' }); + await store.save(view2); + + const loaded = await store.load('overwrite-test'); + assert.strictEqual(loaded.origin, 'private'); + }); +}); diff --git a/src/view/view.js b/src/view/view.js new file mode 100644 index 0000000..2c0b120 --- /dev/null +++ b/src/view/view.js @@ -0,0 +1,52 @@ +/** + * View class - represents a predicate + projection over cached data + */ +import { encodeOrigin } from '@_all_docs/cache'; + +export class View { + constructor({ name, origin, registry, type = 'packument', select = null }) { + if (!name) throw new Error('View name is required'); + if (!origin && !registry) throw new Error('Origin or registry is required'); + + this.name = name; + this.origin = origin || encodeOrigin(registry); + this.registry = registry || null; + this.type = type; + this.select = select; + this.createdAt = new Date().toISOString(); + } + + /** + * Get the cache key prefix for this view's origin and type + */ + getCacheKeyPrefix() { + return `v1:${this.type}:${this.origin}:`; + } + + toJSON() { + return { + name: this.name, + origin: this.origin, + registry: this.registry, + type: this.type, + select: this.select, + createdAt: this.createdAt + }; + } + + static fromJSON(data) { + const view = new View({ + name: data.name, + origin: data.origin, + type: data.type, + select: data.select + }); + view.registry = data.registry; + view.createdAt = data.createdAt; + return view; + } + + toString() { + return `View(${this.name}: ${this.origin}/${this.type})`; + } +} diff --git a/src/view/view.test.js b/src/view/view.test.js new file mode 100644 index 0000000..c25deeb --- /dev/null +++ b/src/view/view.test.js @@ -0,0 +1,104 @@ +import { test, describe } from 'node:test'; +import assert from 'node:assert'; +import { View } from './view.js'; + +describe('View', () => { + test('creates a view with required properties', () => { + const view = new View({ + name: 'test-view', + origin: 'npm' + }); + + assert.strictEqual(view.name, 'test-view'); + assert.strictEqual(view.origin, 'npm'); + assert.strictEqual(view.type, 'packument'); + assert.strictEqual(view.select, null); + assert.ok(view.createdAt); + }); + + test('creates a view with all properties', () => { + const view = new View({ + name: 'full-view', + origin: 'npm', + type: 'partition', + select: 'name, versions|keys' + }); + + assert.strictEqual(view.name, 'full-view'); + assert.strictEqual(view.origin, 'npm'); + assert.strictEqual(view.type, 'partition'); + assert.strictEqual(view.select, 'name, versions|keys'); + }); + + test('encodes registry URL to origin', () => { + const view = new View({ + name: 'registry-view', + registry: 'https://npm.example.com' + }); + + // Origin is truncated: example -> exa + le = exale + assert.strictEqual(view.origin, 'npm.exale.com'); + assert.strictEqual(view.registry, 'https://npm.example.com'); + }); + + test('throws when name is missing', () => { + assert.throws(() => { + new View({ origin: 'npm' }); + }, /View name is required/); + }); + + test('throws when origin and registry are missing', () => { + assert.throws(() => { + new View({ name: 'test' }); + }, /Origin or registry is required/); + }); + + test('generates cache key prefix', () => { + const view = new View({ + name: 'test', + origin: 'npm', + type: 'packument' + }); + + assert.strictEqual(view.getCacheKeyPrefix(), 'v1:packument:npm:'); + }); + + test('serializes to JSON', () => { + const view = new View({ + name: 'test', + origin: 'npm', + select: 'name' + }); + + const json = view.toJSON(); + assert.strictEqual(json.name, 'test'); + assert.strictEqual(json.origin, 'npm'); + assert.strictEqual(json.select, 'name'); + assert.ok(json.createdAt); + }); + + test('deserializes from JSON', () => { + const json = { + name: 'test', + origin: 'npm', + type: 'packument', + select: 'name, versions|keys', + createdAt: '2024-01-15T10:00:00.000Z' + }; + + const view = View.fromJSON(json); + assert.strictEqual(view.name, 'test'); + assert.strictEqual(view.origin, 'npm'); + assert.strictEqual(view.select, 'name, versions|keys'); + assert.strictEqual(view.createdAt, json.createdAt); + }); + + test('toString returns readable representation', () => { + const view = new View({ + name: 'npm-packages', + origin: 'npm' + }); + + assert.strictEqual(view.toString(), 'View(npm-packages: npm/packument)'); + }); +});