diff --git a/app/_components/toolkit-docs/components/available-tools-table.tsx b/app/_components/toolkit-docs/components/available-tools-table.tsx index b0f9af07e..c163d541d 100644 --- a/app/_components/toolkit-docs/components/available-tools-table.tsx +++ b/app/_components/toolkit-docs/components/available-tools-table.tsx @@ -22,7 +22,11 @@ import { import { useEffect, useMemo, useRef, useState } from "react"; import { SCROLLING_CELL } from "../constants"; -import type { AvailableToolsTableProps, SecretType } from "../types"; +import type { + AvailableToolsTableProps, + BehaviorFlagKey, + SecretType, +} from "../types"; import { normalizeScopes } from "./scopes-display"; const DEFAULT_PAGE_SIZE = 25; @@ -236,6 +240,207 @@ export type AvailableToolsSort = | "secrets_first" | "selected_first"; +const BEHAVIOR_FILTER_LABELS: Record = { + readOnly: "Read only", + destructive: "Destructive", + idempotent: "Idempotent", + openWorld: "Open world", +}; + +const BEHAVIOR_FILTER_ORDER: BehaviorFlagKey[] = [ + "readOnly", + "destructive", + "idempotent", + "openWorld", +]; + +function formatMetadataLabel(value: string): string { + const words = value.split("_"); + return words + .map((word, index) => { + if (word === "crm") { + return "CRM"; + } + if (index === 0) { + return word.charAt(0).toUpperCase() + word.slice(1).toLowerCase(); + } + return word.toLowerCase(); + }) + .join(" "); +} + +type BehaviorFlagsState = Partial>; + +function getBehaviorFilterStateLabel(value: boolean | undefined): string { + if (value === undefined) { + return "Any"; + } + if (value) { + return "Yes"; + } + return "No"; +} + +function getNextBehaviorFilterValue( + value: boolean | undefined +): boolean | undefined { + if (value === undefined) { + return true; + } + if (value) { + return false; + } + return; +} + +function getBehaviorFilterToneClass(value: boolean | undefined): string { + if (value === undefined) { + return "border-muted/60 bg-neutral-dark/20 text-muted-foreground hover:bg-neutral-dark/35"; + } + if (value) { + return "border-emerald-500/45 bg-emerald-500/15 text-emerald-200 hover:bg-emerald-500/25"; + } + return "border-rose-500/45 bg-rose-500/15 text-rose-200 hover:bg-rose-500/25"; +} + +function BehaviorFilterButtons({ + behaviorFilterKeys, + behaviorFlags, + onBehaviorFlagsChange, + onFiltersChanged, +}: { + behaviorFilterKeys: BehaviorFlagKey[]; + behaviorFlags: BehaviorFlagsState; + onBehaviorFlagsChange: ( + updater: (current: BehaviorFlagsState) => BehaviorFlagsState + ) => void; + onFiltersChanged: () => void; +}) { + return behaviorFilterKeys.map((key) => { + const currentValue = behaviorFlags[key]; + const stateLabel = getBehaviorFilterStateLabel(currentValue); + + return ( + + ); + }); +} + +function MetadataFilterSection({ + operationOptions, + activeOperations, + onActiveOperationsChange, + behaviorFilterKeys, + behaviorFlags, + onBehaviorFlagsChange, + hasActiveMetadataFilters, + onFiltersChanged, +}: { + operationOptions: string[]; + activeOperations: Set; + onActiveOperationsChange: ( + updater: (current: Set) => Set + ) => void; + behaviorFilterKeys: BehaviorFlagKey[]; + behaviorFlags: BehaviorFlagsState; + onBehaviorFlagsChange: ( + updater: (current: BehaviorFlagsState) => BehaviorFlagsState + ) => void; + hasActiveMetadataFilters: boolean; + onFiltersChanged: () => void; +}) { + return ( +
+
+ {operationOptions.length > 0 && ( +
+ + Operations + + {operationOptions.map((operation) => ( + + ))} +
+ )} + {behaviorFilterKeys.length > 0 && ( +
+ + Behavior + + +
+ )} + {hasActiveMetadataFilters && ( + + )} +
+
+ ); +} + type SecretDisplayItem = { label: string; href?: string; @@ -487,40 +692,85 @@ export function sortTools( } } -function filterTools( +export type FilterToolsOptions = { + activeOperations?: Set; + behaviorFlags?: Partial>; +}; + +function matchesFilterCategory( + tool: AvailableToolsTableProps["tools"][number], + filter: AvailableToolsFilter +): boolean { + const hasScopes = buildScopeDisplayItems(tool.scopes ?? []).length > 0; + const hasSecrets = + (tool.secretsInfo?.length ?? 0) > 0 || (tool.secrets?.length ?? 0) > 0; + + switch (filter) { + case "has_scopes": + return hasScopes; + case "no_scopes": + return !hasScopes; + case "has_secrets": + return hasSecrets; + case "no_secrets": + return !hasSecrets; + default: + return true; + } +} + +function matchesOperations( + tool: AvailableToolsTableProps["tools"][number], + activeOperations: Set +): boolean { + if (activeOperations.size === 0) { + return true; + } + const toolOps = tool.metadata?.behavior?.operations ?? []; + return toolOps.some((op) => activeOperations.has(op)); +} + +function matchesBehaviorFlags( + tool: AvailableToolsTableProps["tools"][number], + behaviorFlags: Partial> +): boolean { + for (const [key, expected] of Object.entries(behaviorFlags) as [ + BehaviorFlagKey, + boolean | undefined, + ][]) { + if (expected === undefined) { + continue; + } + if (tool.metadata?.behavior?.[key] !== expected) { + return false; + } + } + return true; +} + +export function filterTools( tools: AvailableToolsTableProps["tools"], query: string, - filter: AvailableToolsFilter + filter: AvailableToolsFilter, + options: FilterToolsOptions = {} ): AvailableToolsTableProps["tools"] { + const { activeOperations = new Set(), behaviorFlags = {} } = options; const normalizedQuery = query.trim().toLowerCase(); return tools.filter((tool) => { const haystack = [tool.name, tool.qualifiedName, tool.description ?? ""] .join(" ") .toLowerCase(); - const matchesQuery = - normalizedQuery.length === 0 || haystack.includes(normalizedQuery); - - if (!matchesQuery) { + if (normalizedQuery.length > 0 && !haystack.includes(normalizedQuery)) { return false; } - - const hasScopes = buildScopeDisplayItems(tool.scopes ?? []).length > 0; - const hasSecrets = - (tool.secretsInfo?.length ?? 0) > 0 || (tool.secrets?.length ?? 0) > 0; - - switch (filter) { - case "has_scopes": - return hasScopes; - case "no_scopes": - return !hasScopes; - case "has_secrets": - return hasSecrets; - case "no_secrets": - return !hasSecrets; - default: - return true; + if (!matchesFilterCategory(tool, filter)) { + return false; + } + if (!matchesOperations(tool, activeOperations)) { + return false; } + return matchesBehaviorFlags(tool, behaviorFlags); }); } @@ -550,13 +800,61 @@ export function AvailableToolsTable({ const [query, setQuery] = useState(""); const [filter, setFilter] = useState(defaultFilter); const [sort, setSort] = useState("name_asc"); + const [activeOperations, setActiveOperations] = useState>( + () => new Set() + ); + const [behaviorFlags, setBehaviorFlags] = useState({}); const [pageSize, setPageSize] = useState(DEFAULT_PAGE_SIZE); const [page, setPage] = useState(1); + const operationOptions = useMemo(() => { + const values = new Set(); + for (const tool of safeTools) { + for (const operation of tool.metadata?.behavior?.operations ?? []) { + if (operation.trim().length > 0) { + values.add(operation); + } + } + } + return [...values].sort((a, b) => a.localeCompare(b)); + }, [safeTools]); + + const behaviorFilterKeys = useMemo(() => { + const available = new Set(); + for (const tool of safeTools) { + const behavior = tool.metadata?.behavior; + if (!behavior) { + continue; + } + for (const key of BEHAVIOR_FILTER_ORDER) { + if (behavior[key] !== undefined) { + available.add(key); + } + } + } + return BEHAVIOR_FILTER_ORDER.filter((key) => available.has(key)); + }, [safeTools]); + + const hasMetadataFilters = + operationOptions.length > 0 || behaviorFilterKeys.length > 0; + const hasActiveMetadataFilters = + activeOperations.size > 0 || Object.keys(behaviorFlags).length > 0; + const filteredTools = useMemo(() => { - const filtered = filterTools(safeTools, query, filter); + const filtered = filterTools(safeTools, query, filter, { + activeOperations, + behaviorFlags, + }); return sortTools(filtered, sort, selectedTools); - }, [safeTools, query, filter, sort, selectedTools]); + }, [ + safeTools, + query, + filter, + sort, + selectedTools, + activeOperations, + behaviorFlags, + ]); const pageCount = useMemo( () => Math.max(1, Math.ceil(filteredTools.length / pageSize)), @@ -583,107 +881,135 @@ export function AvailableToolsTable({ return (
{(enableSearch || enableFilters) && ( -
- {enableSearch && ( -
- - { - setQuery(event.target.value); - setPage(1); - }} - placeholder={searchPlaceholder} - type="text" - value={query} - /> - {query && ( - + )} +
+ )} + {enableFilters && ( + <> + { - setFilter(value as AvailableToolsFilter); - setPage(1); - }} - value={filter} - > - - - - - All tools - - Requires secrets only - - No secrets required - - + + + + + All tools + + Requires OAuth scopes only + + No OAuth scopes + + Requires secrets only + + + No secrets required + + + + + + + )} + + + {filteredTools.length} + {" "} + of {safeTools.length} tools + +
+ + {enableFilters && hasMetadataFilters && ( + setPage(1)} + operationOptions={operationOptions} + /> )} - - - - - {filteredTools.length} - {" "} - of {safeTools.length} -
)} {filteredTools.length === 0 ? ( diff --git a/app/_components/toolkit-docs/components/dynamic-code-block.tsx b/app/_components/toolkit-docs/components/dynamic-code-block.tsx index 48e8947bc..4d559a43d 100644 --- a/app/_components/toolkit-docs/components/dynamic-code-block.tsx +++ b/app/_components/toolkit-docs/components/dynamic-code-block.tsx @@ -351,26 +351,33 @@ export function DynamicCodeBlock({ return (
{isExpanded ? ( -
-
- +
+
+
+ + Example + +
+ +
-
-
- +
+
+ {selectedLanguage.toLowerCase()} ) : ( )}
diff --git a/app/_components/toolkit-docs/components/index.ts b/app/_components/toolkit-docs/components/index.ts index ff3a14714..1b053d8f1 100644 --- a/app/_components/toolkit-docs/components/index.ts +++ b/app/_components/toolkit-docs/components/index.ts @@ -11,6 +11,10 @@ export { export { DynamicCodeBlock } from "./dynamic-code-block"; export { ParametersTable } from "./parameters-table"; export { ScopesDisplay } from "./scopes-display"; +export { + buildBehaviorRows, + ToolMetadataSection, +} from "./tool-metadata-section"; export { ToolSection } from "./tool-section"; export { ToolkitHeader } from "./toolkit-header"; export { ToolkitPage } from "./toolkit-page"; diff --git a/app/_components/toolkit-docs/components/tool-metadata-section.tsx b/app/_components/toolkit-docs/components/tool-metadata-section.tsx new file mode 100644 index 000000000..58abf8fbd --- /dev/null +++ b/app/_components/toolkit-docs/components/tool-metadata-section.tsx @@ -0,0 +1,226 @@ +"use client"; + +import { Badge } from "@arcadeai/design-system"; +import { Check, ChevronDown, Lightbulb, Minus, X } from "lucide-react"; +import { + TOOL_METADATA_FALLBACK_STYLE, + TOOL_METADATA_OPERATION_STYLES, + TOOL_METADATA_SERVICE_DOMAIN_STYLES, +} from "../constants"; +import type { ToolMetadata, ToolMetadataBehavior } from "../types"; + +type BehaviorFlagKey = "readOnly" | "destructive" | "idempotent" | "openWorld"; + +const BEHAVIOR_LABELS: Record = { + readOnly: "Read only", + destructive: "Destructive", + idempotent: "Idempotent", + openWorld: "Open world", +}; + +const BEHAVIOR_DESCRIPTIONS: Record = { + readOnly: "Does not modify remote state.", + destructive: "May delete or overwrite remote data.", + idempotent: "Safe to retry without extra side effects.", + openWorld: "Can call out to external systems.", +}; + +export type BehaviorRow = { + key: BehaviorFlagKey; + label: string; + value: boolean | null; +}; + +export function buildBehaviorRows( + behavior: ToolMetadataBehavior +): readonly BehaviorRow[] { + return (Object.keys(BEHAVIOR_LABELS) as BehaviorFlagKey[]).map((key) => ({ + key, + label: BEHAVIOR_LABELS[key], + value: behavior[key] ?? null, + })); +} + +function formatEnumLabel(value: string): string { + const words = value.split("_"); + return words + .map((word, index) => { + if (word === "crm") { + return "CRM"; + } + if (index === 0) { + return word.charAt(0).toUpperCase() + word.slice(1).toLowerCase(); + } + return word.toLowerCase(); + }) + .join(" "); +} + +function EnumBadge({ + value, + styles, +}: { + value: string; + styles: Record; +}) { + const styleClass = styles[value] ?? TOOL_METADATA_FALLBACK_STYLE; + return ( + + + {formatEnumLabel(value)} + + ); +} + +function BooleanBadge({ value }: { value: boolean | null }) { + if (value === null) { + return ( + + Unknown + + ); + } + + return value ? ( + + Yes + + ) : ( + + No + + ); +} + +export function ToolMetadataSection({ + metadata, +}: { + metadata?: ToolMetadata | null; +}) { + if (!metadata) { + return null; + } + + const behaviorRows = buildBehaviorRows(metadata.behavior); + const hasOperations = metadata.behavior.operations.length > 0; + const hasServiceDomains = metadata.classification.serviceDomains.length > 0; + const hasAnyBehaviorValue = behaviorRows.some((row) => row.value !== null); + const hasExtras = + metadata.extras != null && Object.keys(metadata.extras).length > 0; + + if ( + !(hasOperations || hasServiceDomains || hasAnyBehaviorValue || hasExtras) + ) { + return null; + } + + return ( +
+
+
+ +
+
+

+ Execution hints +

+

+ Signals for MCP clients and agents about how this tool behaves. +

+
+
+ +
+ {(hasOperations || hasServiceDomains) && ( +
+ {hasOperations && ( +
+ + Operations + +
+ {metadata.behavior.operations.map((operation) => ( + + ))} +
+
+ )} + + {hasServiceDomains && ( +
+ + Service domains + +
+ {metadata.classification.serviceDomains.map((domain) => ( + + ))} +
+
+ )} +
+ )} + + {hasAnyBehaviorValue && ( +
+ + MCP behavior + +
+ {behaviorRows.map((row) => ( +
+
+ + {row.label} + +
+ +
+
+

+ {BEHAVIOR_DESCRIPTIONS[row.key]} +

+
+ ))} +
+
+ )} + + {hasExtras && ( +
+ + Additional metadata + + + 
+              {JSON.stringify(metadata.extras, null, 2)}
+
+
+ )} +
+
+ ); +} diff --git a/app/_components/toolkit-docs/components/tool-section.tsx b/app/_components/toolkit-docs/components/tool-section.tsx index 879459b92..0f22427b9 100644 --- a/app/_components/toolkit-docs/components/tool-section.tsx +++ b/app/_components/toolkit-docs/components/tool-section.tsx @@ -12,6 +12,7 @@ import { import { DynamicCodeBlock } from "./dynamic-code-block"; import { ParametersTable } from "./parameters-table"; import { ScopesDisplay } from "./scopes-display"; +import { ToolMetadataSection } from "./tool-metadata-section"; const COPY_FEEDBACK_MS = 2000; const JSON_PRETTY_PRINT_INDENT = 2; @@ -484,6 +485,7 @@ export function ToolSection({ showSelection={showSelection} tool={tool} /> + +): string | null { + if (tools.length === 0) { + return null; + } + + let sharedDomain: string | null = null; + + for (const tool of tools) { + const serviceDomains = tool.metadata?.classification?.serviceDomains; + if (!serviceDomains || serviceDomains.length !== 1) { + return null; + } + + const domain = serviceDomains[0]; + if (!domain || typeof domain !== "string") { + return null; + } + + if (sharedDomain === null) { + sharedDomain = domain; + continue; + } + + if (sharedDomain !== domain) { + return null; + } + } + + return sharedDomain; +} + function toTitleCaseCategory(category: ToolkitCategory): string { return category .split("-") @@ -557,6 +600,10 @@ export function ToolkitPage({ data }: ToolkitPageProps) { }), [data.id, data.metadata] ); + const sharedServiceDomain = useMemo( + () => getSharedServiceDomain(tools), + [tools] + ); const handleScopeSelectionChange = (toolNames: string[]) => { setSelectedTools(new Set(toolNames)); @@ -583,6 +630,22 @@ export function ToolkitPage({ data }: ToolkitPageProps) {

{data.label}

+ {sharedServiceDomain && ( +
+ + Service domain + + + {sharedServiceDomain.replace(/_/g, " ").toUpperCase()} + +
+ )} diff --git a/app/_components/toolkit-docs/constants.ts b/app/_components/toolkit-docs/constants.ts index 59e86a69f..138aee9df 100644 --- a/app/_components/toolkit-docs/constants.ts +++ b/app/_components/toolkit-docs/constants.ts @@ -176,3 +176,55 @@ export const ICON_SIZES = { stat: "h-4 w-4", inline: "h-3 w-3", } as const; + +// ============================================================================= +// Tool Metadata Style Maps +// ============================================================================= + +/** + * Tailwind classes for each operation enum value. + * Fallback to TOOL_METADATA_FALLBACK_STYLE for unknown values. + */ +export const TOOL_METADATA_OPERATION_STYLES: Record = { + read: "border-blue-500/40 bg-blue-500/10 text-blue-300", + create: "border-emerald-500/40 bg-emerald-500/10 text-emerald-300", + update: "border-amber-500/40 bg-amber-500/10 text-amber-300", + delete: "border-rose-500/40 bg-rose-500/10 text-rose-300", + opaque: "border-violet-500/40 bg-violet-500/10 text-violet-300", +}; + +/** + * Tailwind classes for each service domain enum value. + * Fallback to TOOL_METADATA_FALLBACK_STYLE for unknown values. + */ +export const TOOL_METADATA_SERVICE_DOMAIN_STYLES: Record = { + calendar: "border-blue-400/40 bg-blue-400/10 text-blue-300", + cloud_storage: "border-sky-500/40 bg-sky-500/10 text-sky-300", + code_sandbox: "border-cyan-500/40 bg-cyan-500/10 text-cyan-300", + crm: "border-indigo-500/40 bg-indigo-500/10 text-indigo-300", + customer_support: "border-teal-500/40 bg-teal-500/10 text-teal-300", + documents: "border-lime-500/40 bg-lime-500/10 text-lime-300", + ecommerce: "border-orange-500/40 bg-orange-500/10 text-orange-300", + email: "border-blue-500/40 bg-blue-500/10 text-blue-300", + financial_data: "border-emerald-500/40 bg-emerald-500/10 text-emerald-300", + geospatial: "border-green-500/40 bg-green-500/10 text-green-300", + messaging: "border-sky-500/40 bg-sky-500/10 text-sky-300", + music_streaming: "border-pink-500/40 bg-pink-500/10 text-pink-300", + payments: "border-emerald-400/40 bg-emerald-400/10 text-emerald-300", + presentations: "border-amber-500/40 bg-amber-500/10 text-amber-300", + project_management: "border-cyan-400/40 bg-cyan-400/10 text-cyan-300", + social_media: "border-violet-500/40 bg-violet-500/10 text-violet-300", + source_code: "border-slate-500/40 bg-slate-500/10 text-slate-300", + spreadsheets: "border-lime-400/40 bg-lime-400/10 text-lime-300", + travel: "border-rose-400/40 bg-rose-400/10 text-rose-300", + video_conferencing: + "border-fuchsia-500/40 bg-fuchsia-500/10 text-fuchsia-300", + video_hosting: "border-red-500/40 bg-red-500/10 text-red-300", + web_scraping: "border-zinc-500/40 bg-zinc-500/10 text-zinc-300", +}; + +/** + * Fallback style for enum values not found in the maps above. + */ +export const TOOL_METADATA_FALLBACK_STYLE = + "border-muted/60 bg-muted/20 text-muted-foreground"; diff --git a/app/_components/toolkit-docs/types/index.ts b/app/_components/toolkit-docs/types/index.ts index 423c58813..827e13850 100644 --- a/app/_components/toolkit-docs/types/index.ts +++ b/app/_components/toolkit-docs/types/index.ts @@ -197,6 +197,12 @@ export type ToolMetadataBehavior = { openWorld?: boolean; }; +export type BehaviorFlagKey = + | "readOnly" + | "destructive" + | "idempotent" + | "openWorld"; + export type ToolMetadata = { classification: ToolMetadataClassification; behavior: ToolMetadataBehavior; @@ -449,6 +455,7 @@ export type AvailableToolsTableProps = { secrets?: string[]; secretsInfo?: ToolSecret[]; scopes?: string[]; + metadata?: ToolMetadata | null; }>; /** Optional label for the secrets column */ secretsColumnLabel?: string; diff --git a/package.json b/package.json index f796fcd9b..8528807f0 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,8 @@ "vale:sync": "vale sync", "check-redirects": "pnpm dlx tsx scripts/check-redirects.ts", "update-links": "pnpm dlx tsx scripts/update-internal-links.ts", - "check-meta": "pnpm dlx tsx scripts/check-meta-keys.ts" + "check-meta": "pnpm dlx tsx scripts/check-meta-keys.ts", + "metadata-report": "pnpm dlx tsx toolkit-docs-generator/scripts/report-tool-metadata.ts" }, "repository": { "type": "git", diff --git a/toolkit-docs-generator/scripts/report-tool-metadata.ts b/toolkit-docs-generator/scripts/report-tool-metadata.ts new file mode 100644 index 000000000..51624b8e0 --- /dev/null +++ b/toolkit-docs-generator/scripts/report-tool-metadata.ts @@ -0,0 +1,37 @@ +#!/usr/bin/env node +/** + * CLI script to report tool metadata coverage and distinct enum values. + * Resolves data directory relative to this script, so it works regardless of cwd. + */ + +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { collectToolMetadataStats } from "../src/utils/tool-metadata-audit.ts"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const DATA_DIR = join(__dirname, "..", "data", "toolkits"); + +async function main(): Promise { + const stats = await collectToolMetadataStats({ dataDir: DATA_DIR }); + + console.log("Tool metadata report"); + console.log("==================="); + console.log(`Total tools: ${stats.coverage.totalTools}`); + console.log(`With metadata: ${stats.coverage.withMetadata}`); + console.log( + `Distinct operations: ${stats.distinct.operations.join(", ") || "none"}` + ); + console.log( + `Distinct service domains: ${stats.distinct.serviceDomains.join(", ") || "none"}` + ); + console.log(`Tools with extras: ${stats.extras.toolsWithExtras}`); + if (stats.extras.distinctKeys.length > 0) { + console.log(`Extras keys: ${stats.extras.distinctKeys.join(", ")}`); + } +} + +main().catch((err) => { + console.error("metadata-report failed:", err); + process.exit(1); +}); diff --git a/toolkit-docs-generator/src/utils/tool-metadata-audit.ts b/toolkit-docs-generator/src/utils/tool-metadata-audit.ts new file mode 100644 index 000000000..b3e80cbd0 --- /dev/null +++ b/toolkit-docs-generator/src/utils/tool-metadata-audit.ts @@ -0,0 +1,154 @@ +import { readdir, readFile } from "node:fs/promises"; +import { join } from "node:path"; + +type BooleanCounts = { true: number; false: number; undefined: number }; +type BooleanKey = "readOnly" | "destructive" | "idempotent" | "openWorld"; + +type ToolMeta = { + classification?: { serviceDomains?: string[] }; + behavior?: { + operations?: string[]; + readOnly?: boolean; + destructive?: boolean; + idempotent?: boolean; + openWorld?: boolean; + }; + extras?: Record | null; +} | null; + +type ToolEntry = { metadata?: ToolMeta }; + +export type ToolMetadataStats = { + coverage: { + totalTools: number; + withMetadata: number; + }; + distinct: { + operations: string[]; + serviceDomains: string[]; + }; + booleans: { + readOnly: BooleanCounts; + destructive: BooleanCounts; + idempotent: BooleanCounts; + openWorld: BooleanCounts; + }; + extras: { + toolsWithExtras: number; + distinctKeys: string[]; + }; +}; + +type AccumulatorState = { + operationsSet: Set; + serviceDomainsSet: Set; + extrasKeysSet: Set; + totalTools: number; + withMetadata: number; + toolsWithExtras: number; + booleans: Record; +}; + +function incrementBoolean(counts: BooleanCounts, val: boolean | undefined) { + if (val === true) counts.true++; + else if (val === false) counts.false++; + else counts.undefined++; +} + +function accumulateToolMeta( + meta: NonNullable, + acc: AccumulatorState +) { + for (const op of meta.behavior?.operations ?? []) { + acc.operationsSet.add(op); + } + for (const sd of meta.classification?.serviceDomains ?? []) { + acc.serviceDomainsSet.add(sd); + } + + const b = meta.behavior; + if (b) { + for (const key of [ + "readOnly", + "destructive", + "idempotent", + "openWorld", + ] as const) { + incrementBoolean(acc.booleans[key], b[key]); + } + } + + if (meta.extras && Object.keys(meta.extras).length > 0) { + acc.toolsWithExtras++; + for (const k of Object.keys(meta.extras)) { + acc.extrasKeysSet.add(k); + } + } +} + +function accumulateTools(tools: ToolEntry[], acc: AccumulatorState) { + for (const tool of tools) { + acc.totalTools++; + const meta = tool.metadata; + if (!meta) continue; + acc.withMetadata++; + accumulateToolMeta(meta, acc); + } +} + +async function parseJsonFile( + path: string +): Promise<{ tools?: unknown[] } | null> { + try { + const content = await readFile(path, "utf-8"); + return JSON.parse(content) as { tools?: unknown[] }; + } catch (err) { + console.warn( + `[tool-metadata-audit] Skipping unreadable or malformed JSON: ${path} — ${err}` + ); + return null; + } +} + +export async function collectToolMetadataStats(opts: { + dataDir: string; +}): Promise { + const { dataDir } = opts; + const files = await readdir(dataDir); + const jsonFiles = files.filter((f) => f.endsWith(".json")); + + const acc: AccumulatorState = { + operationsSet: new Set(), + serviceDomainsSet: new Set(), + extrasKeysSet: new Set(), + totalTools: 0, + withMetadata: 0, + toolsWithExtras: 0, + booleans: { + readOnly: { true: 0, false: 0, undefined: 0 }, + destructive: { true: 0, false: 0, undefined: 0 }, + idempotent: { true: 0, false: 0, undefined: 0 }, + openWorld: { true: 0, false: 0, undefined: 0 }, + }, + }; + + for (const file of jsonFiles) { + const data = await parseJsonFile(join(dataDir, file)); + if (!data) continue; + const tools = Array.isArray(data.tools) ? (data.tools as ToolEntry[]) : []; + accumulateTools(tools, acc); + } + + return { + coverage: { totalTools: acc.totalTools, withMetadata: acc.withMetadata }, + distinct: { + operations: Array.from(acc.operationsSet).sort(), + serviceDomains: Array.from(acc.serviceDomainsSet).sort(), + }, + booleans: acc.booleans, + extras: { + toolsWithExtras: acc.toolsWithExtras, + distinctKeys: Array.from(acc.extrasKeysSet).sort(), + }, + }; +} diff --git a/toolkit-docs-generator/tests/app-lib/available-tools-filter-behavior.test.ts b/toolkit-docs-generator/tests/app-lib/available-tools-filter-behavior.test.ts new file mode 100644 index 000000000..9bbb5d5c5 --- /dev/null +++ b/toolkit-docs-generator/tests/app-lib/available-tools-filter-behavior.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from "vitest"; +import { filterTools } from "../../../app/_components/toolkit-docs/components/available-tools-table"; + +const makeTool = ( + name: string, + behavior: Record +) => ({ + name, + qualifiedName: `Test.${name}`, + description: null, + metadata: { + classification: { serviceDomains: [] }, + behavior: { + operations: [], + readOnly: behavior.readOnly, + destructive: behavior.destructive, + idempotent: behavior.idempotent, + openWorld: behavior.openWorld, + }, + extras: null, + }, +}); + +describe("filterTools — behavior flags", () => { + const tools = [ + makeTool("ReadOnly", { readOnly: true, destructive: false }), + makeTool("Destructive", { readOnly: false, destructive: true }), + makeTool("Safe", { readOnly: false, destructive: false }), + { + name: "NoMeta", + qualifiedName: "Test.NoMeta", + description: null, + metadata: null, + }, + ]; + + it("returns all tools when no behavior flags are active", () => { + expect(filterTools(tools, "", "all")).toHaveLength(4); + }); + + it("filters to readOnly=true tools", () => { + const result = filterTools(tools, "", "all", { + behaviorFlags: { readOnly: true }, + }); + expect(result.map((tool) => tool.name)).toEqual(["ReadOnly"]); + }); + + it("filters to destructive=true tools", () => { + const result = filterTools(tools, "", "all", { + behaviorFlags: { destructive: true }, + }); + expect(result.map((tool) => tool.name)).toEqual(["Destructive"]); + }); + + it("ANDs multiple flags together", () => { + const result = filterTools(tools, "", "all", { + behaviorFlags: { readOnly: true, destructive: true }, + }); + expect(result).toHaveLength(0); + }); + + it("ignores behavior flags explicitly set to undefined", () => { + const result = filterTools(tools, "", "all", { + behaviorFlags: { + readOnly: undefined, + }, + }); + expect(result).toHaveLength(4); + }); +}); diff --git a/toolkit-docs-generator/tests/app-lib/available-tools-filter-operations.test.ts b/toolkit-docs-generator/tests/app-lib/available-tools-filter-operations.test.ts new file mode 100644 index 000000000..45b7e26a3 --- /dev/null +++ b/toolkit-docs-generator/tests/app-lib/available-tools-filter-operations.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from "vitest"; +import { filterTools } from "../../../app/_components/toolkit-docs/components/available-tools-table"; + +const makeTool = (name: string, operations: string[]) => ({ + name, + qualifiedName: `Slack.${name}`, + description: null, + metadata: { + classification: { serviceDomains: [] }, + behavior: { + operations, + readOnly: false, + destructive: false, + idempotent: false, + openWorld: false, + }, + extras: null, + }, +}); + +describe("filterTools — operations", () => { + const tools = [ + makeTool("GetMessages", ["read"]), + makeTool("PostMessage", ["create"]), + makeTool("DeleteMessage", ["delete"]), + { + name: "NoMeta", + qualifiedName: "Slack.NoMeta", + description: null, + metadata: null, + }, + ]; + + it("returns all tools when no operations are selected", () => { + expect(filterTools(tools, "", "all")).toHaveLength(4); + }); + + it("returns only tools matching selected operations", () => { + const result = filterTools(tools, "", "all", { + activeOperations: new Set(["read"]), + }); + expect(result.map((tool) => tool.name)).toEqual(["GetMessages"]); + }); + + it("returns a union when multiple operations are selected", () => { + const result = filterTools(tools, "", "all", { + activeOperations: new Set(["read", "create"]), + }); + expect(result.map((tool) => tool.name)).toEqual([ + "GetMessages", + "PostMessage", + ]); + }); + + it("does not include tools without metadata when operation filters are active", () => { + const result = filterTools(tools, "", "all", { + activeOperations: new Set(["read"]), + }); + expect(result.map((tool) => tool.name)).not.toContain("NoMeta"); + }); +}); diff --git a/toolkit-docs-generator/tests/app-lib/shared-service-domain.test.ts b/toolkit-docs-generator/tests/app-lib/shared-service-domain.test.ts new file mode 100644 index 000000000..672d03503 --- /dev/null +++ b/toolkit-docs-generator/tests/app-lib/shared-service-domain.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from "vitest"; +import { getSharedServiceDomain } from "../../../app/_components/toolkit-docs/components/toolkit-page"; + +const makeTool = (domains: string[]) => ({ + metadata: { + classification: { serviceDomains: domains }, + behavior: { operations: [] }, + extras: null, + }, +}); + +describe("getSharedServiceDomain", () => { + it("returns the domain when all tools share exactly one", () => { + const tools = [ + makeTool(["messaging"]), + makeTool(["messaging"]), + makeTool(["messaging"]), + ]; + expect(getSharedServiceDomain(tools)).toBe("messaging"); + }); + + it("returns null when tools have different domains", () => { + const tools = [makeTool(["messaging"]), makeTool(["email"])]; + expect(getSharedServiceDomain(tools)).toBeNull(); + }); + + it("returns null when any tool has no metadata", () => { + const tools = [makeTool(["messaging"]), { metadata: null }]; + expect( + getSharedServiceDomain( + tools as Parameters[0] + ) + ).toBeNull(); + }); + + it("returns null for an empty tool list", () => { + expect(getSharedServiceDomain([])).toBeNull(); + }); + + it("returns null when a tool has multiple domains", () => { + const tools = [makeTool(["messaging", "email"]), makeTool(["messaging"])]; + expect(getSharedServiceDomain(tools)).toBeNull(); + }); + + it("returns null when domain is not a string", () => { + const tools = [ + { + metadata: { + classification: { serviceDomains: [123] }, + behavior: { operations: [] }, + extras: null, + }, + }, + makeTool(["messaging"]), + ]; + expect( + getSharedServiceDomain( + tools as Parameters[0] + ) + ).toBeNull(); + }); +});