Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
93 changes: 17 additions & 76 deletions src/agent-mapper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,14 @@ import { pathExists } from "./fs.js";
import { runCommandArgs } from "./exec.js";
import { mapFeatureSeeds, MapResult } from "./mapper.js";
import { FeatureSeed, SeedFileRef, SeedTestRef } from "./mappers/types.js";
import { isSafeFile, normalize, shouldSkip, walk } from "./mappers/shared.js";
import {
applyPathFilters,
isSafeFile,
normalize,
PathFilters,
shouldSkip,
walk,
} from "./mappers/shared.js";

type AgentMapMode = "heuristic" | "auto" | "agent";

Expand All @@ -26,15 +33,10 @@ type AgentMapOptions = {
source: AgentMapMode;
provider: Provider | null;
providerOptions: ProviderOptions;
inventory?: InventoryFilters;
inventory?: PathFilters;
onProgress?: (event: string, fields: Record<string, string | number | boolean>) => void;
};

type InventoryFilters = {
include: string[];
exclude: string[];
};

type RepoInventorySummary = {
files: number;
sourceFiles: number;
Expand Down Expand Up @@ -148,6 +150,7 @@ export async function mapWithSource(
options.provider,
options.providerOptions,
inventory,
options.inventory,
);
options.onProgress?.("agent-done", {
features: agent.features.length,
Expand Down Expand Up @@ -199,6 +202,7 @@ async function agentMap(
provider: Provider,
providerOptions: ProviderOptions,
inventory: RepoInventory,
filters: PathFilters | undefined,
): Promise<MapResult> {
const prompt = buildAgentMapPrompt(project, {
manifests: inventory.manifests,
Expand All @@ -212,12 +216,10 @@ async function agentMap(
const seeds = await Promise.all(
output.features.map((feature) => toSeed(root, feature, inventory.allFiles)),
);
return mapFeatureSeeds(
root,
project,
existing,
uniqueSeeds(seeds.filter((seed): seed is FeatureSeed => seed !== null)),
);
const mappedSeeds = uniqueSeeds(seeds.filter((seed): seed is FeatureSeed => seed !== null));
return filters === undefined
? mapFeatureSeeds(root, project, existing, mappedSeeds)
: mapFeatureSeeds(root, project, existing, mappedSeeds, { filters });
}

async function toSeed(
Expand Down Expand Up @@ -385,10 +387,10 @@ async function repoInventory(
root: string,
project: ProjectRecord,
features: FeatureRecord[],
filters: InventoryFilters | undefined,
filters: PathFilters | undefined,
): Promise<RepoInventory> {
const skipPath = await inventorySkipPath(root, project, features);
const files = applyInventoryFilters(
const files = applyPathFilters(
((await gitInventoryFiles(root)) ?? (await walk(root, [""], skipPath))).filter(
(path) => !skipPath(path),
),
Expand Down Expand Up @@ -443,73 +445,12 @@ async function gitInventoryFiles(root: string): Promise<string[] | null> {
return existing.filter((path): path is string => path !== null);
}

function applyInventoryFilters(files: string[], filters: InventoryFilters | undefined): string[] {
if (filters === undefined) {
return files;
}
return files.filter(
(file) =>
filters.include.some((pattern) => inventoryPatternMatches(pattern, file)) &&
!filters.exclude.some((pattern) => inventoryPatternMatches(pattern, file)),
);
}

function isInventoryPath(path: string): boolean {
return (
path.length > 0 && !isAbsolute(path) && !path.includes("\0") && !path.split("/").includes("..")
);
}

function inventoryPatternMatches(pattern: string, path: string): boolean {
const normalized = pattern.replace(/\\/gu, "/").replace(/^\.\//u, "");
if (normalized === "**" || normalized === "**/*") {
return true;
}
if (normalized.length === 0) {
return false;
}
if (!/[?*]/u.test(normalized)) {
return path === normalized || path.startsWith(`${normalized}/`);
}
if (normalized.endsWith("/**")) {
const prefix = normalized.slice(0, -3);
if (/[?*]/u.test(prefix)) {
return new RegExp(`^${globPatternRegExp(prefix)}(?:/.*)?$`, "u").test(path);
}
return prefix.length === 0 || path === prefix || path.startsWith(`${prefix}/`);
}
return new RegExp(`^${globPatternRegExp(normalized)}$`, "u").test(path);
}

function globPatternRegExp(pattern: string): string {
let source = "";
for (let index = 0; index < pattern.length; index += 1) {
const char = pattern[index];
if (char === "*") {
if (pattern[index + 1] === "*") {
if (pattern[index + 2] === "/") {
source += "(?:.*/)?";
index += 2;
} else {
source += ".*";
index += 1;
}
} else {
source += "[^/]*";
}
} else if (char === "?") {
source += "[^/]";
} else {
source += regexpEscape(char ?? "");
}
}
return source;
}

function regexpEscape(value: string): string {
return value.replace(/[\\^$.*+?()[\]{}|]/gu, "\\$&");
}

async function inventorySkipPath(
root: string,
project: ProjectRecord,
Expand Down
4 changes: 3 additions & 1 deletion src/app.ts
Original file line number Diff line number Diff line change
Expand Up @@ -123,12 +123,14 @@ export async function mapCommand(
const config = applyProviderFlags(loaded.config, flags);
const provider = source === "heuristic" ? null : providerByName(config.provider.name);
const existing = await readFeatures(loaded.paths);
const filters = { include: config.include, exclude: config.exclude };
emitProgress(context, "map", "start", {
source,
existing: existing.length,
dryRun: flags["dryRun"] === true,
});
const heuristic = await mapFeatures(loaded.root, loaded.project, existing, {
filters,
onProgress: (event) => {
emitProgress(context, "map", event.event, {
mapper: event.mapper,
Expand All @@ -149,7 +151,7 @@ export async function mapCommand(
source,
provider,
providerOptions: providerOptions(config),
inventory: { include: config.include, exclude: config.exclude },
inventory: filters,
onProgress: (event, fields) => {
emitProgress(context, "map", event, fields);
},
Expand Down
31 changes: 31 additions & 0 deletions src/mapper.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,37 @@ import { turboTaskGraph } from "./mappers/turbo.js";
import { fixtureRoot, writeFixture } from "./test-helpers.js";

describe("mapFeatures", () => {
it("applies configured path excludes to heuristic feature mapping", async () => {
const root = await fixtureRoot("clawpatch-map-exclude-");
await writeFixture(root, "requirements.txt", "pytest\n");
await writeFixture(root, "src/app/api_service.py", "def call_api(): pass\n");
for (let index = 0; index < 13; index += 1) {
await writeFixture(
root,
`src/client/generated/models/model_${index}.py`,
`class Model${index}: pass\n`,
);
}

const project = await detectProject(root);
const result = await mapFeatures(root, project, [], {
filters: {
include: ["**/*"],
exclude: ["src/client/generated/**"],
},
});
const featurePaths = result.features.flatMap((feature) => [
...feature.entrypoints.map((entrypoint) => entrypoint.path),
...feature.ownedFiles.map((file) => file.path),
...feature.contextFiles.map((file) => file.path),
...feature.tests.map((test) => test.path),
]);

expect(featurePaths).toContain("src/app/api_service.py");
expect(result.features.some((feature) => feature.title.includes("generated"))).toBe(false);
expect(featurePaths.some((path) => path.startsWith("src/client/generated/"))).toBe(false);
});

it("maps package bins, scripts, configs, and Next routes", async () => {
const root = await fixtureRoot("clawpatch-map-");
await writeFixture(
Expand Down
65 changes: 60 additions & 5 deletions src/mapper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import { reactSeeds } from "./mappers/react.js";
import { discoverNodeProjects } from "./mappers/projects.js";
import { rubySeeds } from "./mappers/ruby.js";
import { rustSeeds } from "./mappers/rust.js";
import { nearbyTests } from "./mappers/shared.js";
import { nearbyTests, PathFilters, pathMatchesFilters } from "./mappers/shared.js";
import { swiftSeeds } from "./mappers/swift.js";
import { turboTaskGraph } from "./mappers/turbo.js";
import { FeatureMapper, FeatureSeed, MapperContext } from "./mappers/types.js";
Expand All @@ -38,6 +38,7 @@ export type MapProgressEvent = {

export type MapOptions = {
onProgress?: (event: MapProgressEvent) => void;
filters?: PathFilters;
};

const featureMappers: FeatureMapper[] = [
Expand Down Expand Up @@ -66,21 +67,26 @@ export async function mapFeatures(
options: MapOptions = {},
): Promise<MapResult> {
const seeds = await collectSeeds(root, options);
return mapFeatureSeeds(root, project, existing, seeds);
return mapFeatureSeeds(root, project, existing, seeds, options);
}

export async function mapFeatureSeeds(
root: string,
project: ProjectRecord,
existing: FeatureRecord[],
seeds: FeatureSeed[],
options: MapOptions = {},
): Promise<MapResult> {
const existingById = new Map(existing.map((feature) => [feature.featureId, feature]));
const features: FeatureRecord[] = [];
let created = 0;
let changed = 0;
const now = nowIso();
for (const seed of seeds) {
for (const rawSeed of seeds) {
const seed = filterSeed(rawSeed, options.filters);
if (seed === null) {
continue;
}
const identity = featureIdentity(seed, existingById);
const featureId = identity.featureId;
const previous = existingById.get(featureId);
Expand All @@ -98,9 +104,11 @@ export async function mapFeatureSeeds(
(name): name is string => typeof name === "string",
),
);
const tests = uniqueTests([...(seed.tests ?? []), ...discoveredTests]);
const tests = uniqueTests(
filterTests([...(seed.tests ?? []), ...discoveredTests], options.filters),
);
const contextFiles = uniqueFileRefs([
...(seed.contextFiles ?? []),
...filterFileRefs(seed.contextFiles ?? [], options.filters),
...tests.map((test) => ({ path: test.path, reason: "nearby test" })),
]);
const feature: FeatureRecord = {
Expand Down Expand Up @@ -157,6 +165,53 @@ export async function mapFeatureSeeds(
};
}

function filterSeed(seed: FeatureSeed, filters: PathFilters | undefined): FeatureSeed | null {
if (filters === undefined) {
return seed;
}
const ownedFiles =
seed.ownedFiles === undefined ? undefined : filterFileRefs(seed.ownedFiles, filters);
if (seed.ownedFiles !== undefined && ownedFiles?.length === 0) {
return null;
}
const entryPath = pathMatchesFilters(seed.entryPath, filters)
? seed.entryPath
: (ownedFiles?.[0]?.path ?? null);
if (entryPath === null) {
return null;
}
const filteredSeed = {
...seed,
entryPath,
contextFiles: filterFileRefs(seed.contextFiles ?? [], filters),
tests: filterTests(seed.tests ?? [], filters),
};
if (ownedFiles === undefined) {
return filteredSeed;
}
return { ...filteredSeed, ownedFiles };
}

function filterFileRefs(
refs: Array<{ path: string; reason: string }>,
filters: PathFilters | undefined,
): Array<{ path: string; reason: string }> {
if (filters === undefined) {
return refs;
}
return refs.filter((ref) => pathMatchesFilters(ref.path, filters));
}

function filterTests(
tests: Array<{ path: string; command: string | null }>,
filters: PathFilters | undefined,
): Array<{ path: string; command: string | null }> {
if (filters === undefined) {
return tests;
}
return tests.filter((test) => pathMatchesFilters(test.path, filters));
}

function featureIdentity(
seed: FeatureSeed,
existingById: Map<string, FeatureRecord>,
Expand Down
Loading