feat(views) Relax! 🛋️ views are a useful analysis tool for datasets of unstructured metadata

cli/cli/package.json

@@ -18,6 +18,7 @@
     "@_all_docs/frame": "workspace:*",
     "@_all_docs/packument": "workspace:*",
     "@_all_docs/partition": "workspace:*",
+    "@_all_docs/view": "workspace:*",
     "@_all_docs/worker": "workspace:*",
     "@vltpkg/error-cause": "0.0.0-9",
     "debug": "^4.4.0",

cli/cli/src/cmd/view/define.js

@@ -0,0 +1,99 @@
+import { View, ViewStore } from '@_all_docs/view';
+import { encodeOrigin } from '@_all_docs/cache';
+
+export const usage = `Usage: _all_docs view define <name> [options]
+
+Define a named view over cached registry data.
+
+A view is a predicate (origin filter) plus a projection (field selection).
+Views are stored as JSON files and can be queried or joined.
+
+Options:
+  --origin <key>       Origin key (e.g., npm, paces.exale.com~javpt)
+  --registry <url>     Registry URL (converted to origin key internally)
+  --type <type>        Entity type: packument (default) or partition
+  --select <expr>      Field selection expression
+
+Select Expression Syntax:
+  Simple fields:       name, version, description
+  Nested fields:       time.modified, repository.url
+  With transforms:     versions|keys, dependencies|length
+  With aliases:        versions|keys as version_list
+
+Available Transforms:
+  keys, values         Object to array
+  length               Array/string length
+  first, last          First/last element
+  sort, reverse        Sort/reverse array
+  unique, compact      Dedupe/remove nulls
+  flatten              Flatten nested arrays
+
+Examples:
+  _all_docs view define npm-packages --origin npm
+  _all_docs view define npm-versions --origin npm --select 'name, versions|keys as versions, time'
+  _all_docs view define private --registry https://npm.company.com --select 'name, versions|keys'
+`;
+
+export const command = async (cli) => {
+  if (cli.values.help) {
+    console.log(usage);
+    return;
+  }
+
+  const name = cli._[0];
+  if (!name) {
+    console.error('Error: View name required');
+    console.error('Usage: _all_docs view define <name> --origin <key> [--select <expr>]');
+    process.exit(1);
+  }
+
+  // Validate name
+  if (!/^[a-zA-Z][a-zA-Z0-9_-]*$/.test(name)) {
+    console.error('Error: View name must start with a letter and contain only letters, numbers, underscores, and hyphens');
+    process.exit(1);
+  }
+
+  // Prioritize --registry if specified, otherwise use --origin
+  // Note: cli.values.origin has a default from jack.js, so we check registry first
+  let origin;
+  if (cli.values.registry) {
+    origin = encodeOrigin(cli.values.registry);
+  } else if (cli.values.origin && cli.values.origin !== 'https://replicate.npmjs.com') {
+    // User explicitly set --origin (not the default)
+    origin = cli.values.origin;
+  } else {
+    console.error('Error: --origin or --registry required');
+    console.error('Example: _all_docs view define my-view --origin npm');
+    process.exit(1);
+  }
+
+  const view = new View({
+    name,
+    origin,
+    registry: cli.values.registry || null,
+    type: cli.values.type || 'packument',
+    select: cli.values.select || null
+  });
+
+  const store = new ViewStore(cli.dir('config'));
+
+  // Check if view already exists
+  if (await store.exists(name)) {
+    if (!cli.values.force) {
+      console.error(`Error: View '${name}' already exists. Use --force to overwrite.`);
+      process.exit(1);
+    }
+  }
+
+  await store.save(view);
+
+  console.log(`View '${name}' defined:`);
+  console.log(`  Origin: ${origin}`);
+  if (cli.values.registry) {
+    console.log(`  Registry: ${cli.values.registry}`);
+  }
+  console.log(`  Type: ${view.type}`);
+  if (view.select) {
+    console.log(`  Select: ${view.select}`);
+  }
+};

cli/cli/src/cmd/view/delete.js

@@ -0,0 +1,37 @@
+import { ViewStore } from '@_all_docs/view';
+
+export const usage = `Usage: _all_docs view delete <name>
+
+Delete a defined view.
+
+Options:
+  --force    Skip confirmation
+
+Examples:
+  _all_docs view delete old-view
+  _all_docs view delete old-view --force
+`;
+
+export const command = async (cli) => {
+  if (cli.values.help) {
+    console.log(usage);
+    return;
+  }
+
+  const name = cli._[0];
+  if (!name) {
+    console.error('Error: View name required');
+    console.error('Usage: _all_docs view delete <name>');
+    process.exit(1);
+  }
+
+  const store = new ViewStore(cli.dir('config'));
+
+  if (!await store.exists(name)) {
+    console.error(`Error: View '${name}' does not exist`);
+    process.exit(1);
+  }
+
+  await store.delete(name);
+  console.log(`View '${name}' deleted`);
+};

cli/cli/src/cmd/view/index.js

@@ -0,0 +1,32 @@
+/**
+ * View commands - define, query, and join views over cached data
+ */
+export { command as define, usage as defineUsage } from './define.js';
+export { command as list, usage as listUsage } from './list.js';
+export { command as show, usage as showUsage } from './show.js';
+export { command as deleteView, usage as deleteUsage } from './delete.js';
+export { command as query, usage as queryUsage } from './query.js';
+export { command as join, usage as joinUsage } from './join.js';
+
+export const usage = `Usage: _all_docs view <command> [options]
+
+Manage named views over cached registry data.
+
+Commands:
+  define <name>              Define a new view
+  list                       List all defined views
+  show <name>                Show view details
+  delete <name>              Delete a view
+  query <name>               Query a view (output ndjson)
+  join <left> <right>        Join two views
+
+A view is a predicate (origin filter) plus a projection (field selection).
+Views enable efficient queries and joins across different registry caches.
+
+Examples:
+  _all_docs view define npm-pkgs --origin npm
+  _all_docs view define npm-vers --origin npm --select 'name, versions|keys'
+  _all_docs view list
+  _all_docs view query npm-vers --limit 100
+  _all_docs view join npm-vers cgr-vers --diff
+`;

cli/cli/src/cmd/view/join.js

@@ -0,0 +1,85 @@
+import { ViewStore, joinViews, diffViews } from '@_all_docs/view';
+import { Cache, createStorageDriver } from '@_all_docs/cache';
+
+export const usage = `Usage: _all_docs view join <left-view> <right-view> [options]
+
+Join two views on their common key (package name).
+
+Join Types:
+  --left     Include all from left, matching from right (default)
+  --inner    Only include records present in both views
+  --right    Include all from right, matching from left
+  --full     Include all records from both views
+  --diff     Output records in left but not in right
+
+Options:
+  --on <field>     Join key field (default: name)
+  --limit <n>      Maximum records to return
+  --json           Output as ndjson (default)
+
+Examples:
+  _all_docs view join npm-packages cgr-packages
+  _all_docs view join npm-packages cgr-packages --inner
+  _all_docs view join npm-packages cgr-packages --diff
+  _all_docs view join npm-versions cgr-versions --limit 1000
+`;
+
+export const command = async (cli) => {
+  if (cli.values.help) {
+    console.log(usage);
+    return;
+  }
+
+  const leftName = cli._[0];
+  const rightName = cli._[1];
+
+  if (!leftName || !rightName) {
+    console.error('Error: Two view names required');
+    console.error('Usage: _all_docs view join <left-view> <right-view>');
+    process.exit(1);
+  }
+
+  const store = new ViewStore(cli.dir('config'));
+
+  let leftView, rightView;
+  try {
+    leftView = await store.load(leftName);
+    rightView = await store.load(rightName);
+  } catch (err) {
+    console.error(`Error: ${err.message}`);
+    process.exit(1);
+  }
+
+  const driver = await createStorageDriver({ CACHE_DIR: cli.dir('packuments') });
+  const cache = new Cache({ path: cli.dir('packuments'), driver });
+
+  // Determine join type
+  let type = 'left';
+  if (cli.values.inner) type = 'inner';
+  else if (cli.values.right) type = 'right';
+  else if (cli.values.full) type = 'full';
+
+  const options = {
+    type,
+    on: cli.values.on || 'name',
+    limit: cli.values.limit ? parseInt(cli.values.limit, 10) : undefined
+  };
+
+  try {
+    // Special case for diff
+    if (cli.values.diff) {
+      for await (const record of diffViews(leftView, rightView, cache, options)) {
+        console.log(JSON.stringify(record));
+      }
+      return;
+    }
+
+    // Regular join
+    for await (const record of joinViews(leftView, rightView, cache, options)) {
+      console.log(JSON.stringify(record));
+    }
+  } catch (err) {
+    console.error(`Error joining views: ${err.message}`);
+    process.exit(1);
+  }
+};

cli/cli/src/cmd/view/list.js

@@ -0,0 +1,59 @@
+import { ViewStore } from '@_all_docs/view';
+
+export const usage = `Usage: _all_docs view list
+
+List all defined views.
+
+Options:
+  --json    Output as JSON array
+
+Examples:
+  _all_docs view list
+  _all_docs view list --json
+`;
+
+export const command = async (cli) => {
+  if (cli.values.help) {
+    console.log(usage);
+    return;
+  }
+
+  const store = new ViewStore(cli.dir('config'));
+  const names = await store.list();
+
+  if (names.length === 0) {
+    console.log('No views defined.');
+    console.log('');
+    console.log('Create a view with:');
+    console.log('  _all_docs view define <name> --origin <key> [--select <expr>]');
+    return;
+  }
+
+  if (cli.values.json) {
+    const views = [];
+    for (const name of names) {
+      const view = await store.load(name);
+      views.push(view.toJSON());
+    }
+    console.log(JSON.stringify(views, null, 2));
+    return;
+  }
+
+  console.log('Defined views:');
+  console.log('');
+
+  for (const name of names) {
+    try {
+      const view = await store.load(name);
+      console.log(`  ${name}`);
+      console.log(`    Origin: ${view.origin}`);
+      console.log(`    Type: ${view.type}`);
+      if (view.select) {
+        console.log(`    Select: ${view.select}`);
+      }
+      console.log('');
+    } catch (err) {
+      console.log(`  ${name} (error loading: ${err.message})`);
+    }
+  }
+};