[APPS][Connections Part 6] Build backend module graph during backend builds

packages/plugins/apps/src/backend/ast-parsing/action-catalog-imports.ts

@@ -2,7 +2,9 @@
 // This product includes software developed at Datadog (https://www.datadoghq.com/).
 // Copyright 2019-Present Datadog, Inc.
 
-import type { BaseNode, Program } from 'estree';
+import type { Program } from 'estree';
+
+import { isTypeOnly } from './type-guards';
 
 const ACTION_CATALOG_PACKAGE = '@datadog/action-catalog';
 
@@ -11,8 +13,6 @@ export interface ActionCatalogImports {
     namespaces: Set<string>;
 }
 
-type NodeWithOptionalImportKind = BaseNode & { importKind?: string };
-
 export function collectActionCatalogImports(ast: Program): ActionCatalogImports {
     const functions = new Set<string>();
     const namespaces = new Set<string>();
@@ -51,7 +51,3 @@ function isActionCatalogSource(source: unknown): boolean {
         (source === ACTION_CATALOG_PACKAGE || source.startsWith(`${ACTION_CATALOG_PACKAGE}/`))
     );
 }
-
-function isTypeOnly(node: NodeWithOptionalImportKind): boolean {
-    return node.importKind === 'type';
-}

packages/plugins/apps/src/backend/ast-parsing/extract-backend-functions.ts

@@ -4,7 +4,7 @@
 
 import type { BaseNode, Declaration, Expression, Program } from 'estree';
 
-import { isProgramNode } from './type-guards';
+import { ensureProgram } from './type-guards';
 import type { BackendExport } from './types';
 
 /**
@@ -23,17 +23,13 @@ export function extractExportedFunctions(ast: BaseNode, filePath: string): strin
 }
 
 export function enumerateBackendExports(ast: BaseNode, filePath: string): BackendExport[] {
-    if (!isProgramNode(ast)) {
-        throw new Error(
-            `Expected a Program node from this.parse() for ${filePath}, got ${ast.type}`,
-        );
-    }
+    const program = ensureProgram(ast, filePath);
 
     // Build a map of top-level declarations so we can validate export specifiers.
-    const declarations = buildDeclarationMap(ast);
+    const declarations = buildDeclarationMap(program);
 
     const backendExports: BackendExport[] = [];
-    for (const node of ast.body) {
+    for (const node of program.body) {
         // handles: export default ...
         if (node.type === 'ExportDefaultDeclaration') {
             throw new Error(

packages/plugins/apps/src/backend/ast-parsing/extract-connection-ids.ts

@@ -8,30 +8,22 @@ import {
     analyzeActionCatalogScopes,
     findActionCatalogCallSites,
 } from './action-catalog-call-sites';
-import { collectActionCatalogImports, hasActionCatalogImports } from './action-catalog-imports';
+import { collectActionCatalogImports } from './action-catalog-imports';
 import {
     collectSameModuleConnectionIdBindings,
     extractConnectionIdFromActionCall,
 } from './connection-id-values';
-import { isProgramNode } from './type-guards';
+import { ensureProgram } from './type-guards';
 
 export function extractConnectionIds(ast: BaseNode, filePath: string): string[] {
-    if (!isProgramNode(ast)) {
-        throw new Error(
-            `Expected a Program node from this.parse() for ${filePath}, got ${ast.type}`,
-        );
-    }
-
-    const imports = collectActionCatalogImports(ast);
-    if (!hasActionCatalogImports(imports)) {
-        return [];
-    }
+    const program = ensureProgram(ast, filePath);
 
-    const scopeAnalysis = analyzeActionCatalogScopes(ast, imports);
-    const bindings = collectSameModuleConnectionIdBindings(ast, scopeAnalysis);
+    const imports = collectActionCatalogImports(program);
+    const scopeAnalysis = analyzeActionCatalogScopes(program, imports);
+    const bindings = collectSameModuleConnectionIdBindings(program, scopeAnalysis);
     const connectionIds = new Set<string>();
 
-    for (const callSite of findActionCatalogCallSites(ast, scopeAnalysis, filePath)) {
+    for (const callSite of findActionCatalogCallSites(program, scopeAnalysis, filePath)) {
         const connectionId = extractConnectionIdFromActionCall(
             callSite,
             bindings,

packages/plugins/apps/src/backend/ast-parsing/module-graph.test.ts

@@ -0,0 +1,94 @@
+// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2019-Present Datadog, Inc.
+
+import { parseAst } from 'rollup/parseAst';
+
+import { createParsedModuleRecord } from './module-graph';
+
+const buildRoot = '/project';
+
+describe('Backend Functions - module graph records', () => {
+    test('Should create graph records for app-local backend modules', () => {
+        const record = createParsedModuleRecord(
+            '/project/src/backend/actions.backend.js',
+            buildRoot,
+            parseAst(`
+                import { getEcho } from './helpers/http.js';
+                export function run() {
+                    return getEcho();
+                }
+            `),
+            ['/project/src/backend/helpers/http.js'],
+        );
+
+        expect(record).toMatchObject({
+            id: '/project/src/backend/actions.backend.js',
+            staticDependencies: ['/project/src/backend/helpers/http.js'],
+            unsupportedDependencies: [],
+        });
+        expect(record?.ast.type).toBe('Program');
+    });
+
+    test.each([
+        { description: 'package modules', id: '/project/node_modules/package/index.js' },
+        { description: 'Yarn package cache modules', id: '/project/.yarn/cache/package/index.js' },
+        { description: 'files outside buildRoot', id: '/external/helper.js' },
+        { description: 'non-JavaScript files', id: '/project/src/backend/data.json' },
+    ])('Should skip $description', ({ id }) => {
+        expect(
+            createParsedModuleRecord(id, buildRoot, parseAst('export const value = true;')),
+        ).toBeNull();
+    });
+
+    test.each([
+        '/project/src/backend/helper.mts',
+        '/project/src/backend/helper.cts',
+        '/project/build/helper.js',
+        '/project/dist/helper.js',
+        '/project/.vite/helper.js',
+    ])('Should parse supported app-local module path %s', (id) => {
+        expect(
+            createParsedModuleRecord(id, buildRoot, parseAst('export const value = true;')),
+        ).toEqual(expect.objectContaining({ id }));
+    });
+
+    test.each([
+        {
+            description: 'dynamic local imports',
+            code: "import('./helper.js');",
+            expected: { kind: 'dynamic-import', specifier: './helper.js' },
+        },
+        {
+            description: 'non-literal dynamic imports',
+            code: 'import(helperPath);',
+            expected: { kind: 'dynamic-import', specifier: 'non-literal dynamic import' },
+        },
+        {
+            description: 'local require calls',
+            code: "require('./helper.js');",
+            expected: { kind: 'require', specifier: './helper.js' },
+        },
+    ])('Should record unsupported $description', ({ code, expected }) => {
+        const record = createParsedModuleRecord(
+            '/project/src/backend/actions.backend.js',
+            buildRoot,
+            parseAst(code),
+        );
+
+        expect(record?.unsupportedDependencies).toEqual([expected]);
+    });
+
+    test('Should ignore package dynamic imports and require calls', () => {
+        const record = createParsedModuleRecord(
+            '/project/src/backend/actions.backend.js',
+            buildRoot,
+            parseAst(`
+                import('package');
+                require('package');
+            `),
+        );
+
+        expect(record?.unsupportedDependencies).toEqual([]);
+    });
+});

packages/plugins/apps/src/backend/ast-parsing/module-graph.ts

@@ -0,0 +1,185 @@
+// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2019-Present Datadog, Inc.
+
+import type { BaseNode, ImportExpression, Program, SimpleCallExpression } from 'estree';
+import path from 'path';
+
+import { BACKEND_CODE_EXTENSIONS } from '../../constants';
+
+import { ensureProgram, isStringLiteral } from './type-guards';
+import { walkAst } from './walk-ast';
+
+export interface ParsedModuleRecord {
+    id: string;
+    ast: Program;
+    staticDependencies: string[];
+    unsupportedDependencies: ModuleDependency[];
+}
+
+export interface ModuleDependency {
+    specifier: string;
+    kind: 'dynamic-import' | 'require';
+}
+
+type ImportCallExpression = SimpleCallExpression & { callee: { type: 'Import' } };
+
+const PACKAGE_MANAGER_DIRS = new Set(['node_modules', '.yarn']);
+
+/**
+ * Creates the per-module analysis record consumed by backend-entry reachability
+ * analysis. The caller supplies canonical module IDs and already-resolved
+ * static dependency IDs instead of asking this module to resolve/load files.
+ *
+ * Returns null when the module is outside the analyzable app-local backend
+ * graph, allowing build collectors to skip package/generated modules without
+ * duplicating graph filtering rules.
+ */
+export function createParsedModuleRecord(
+    moduleId: string,
+    buildRoot: string,
+    ast: BaseNode,
+    staticDependencies: string[] = [],
+): ParsedModuleRecord | null {
+    if (!shouldTraverseCollectedModule(moduleId, buildRoot)) {
+        return null;
+    }
+
+    const program = ensureProgram(ast, moduleId);
+
+    return {
+        id: moduleId,
+        ast: program,
+        staticDependencies,
+        unsupportedDependencies: collectUnsupportedModuleDependencies(program),
+    };
+}
+
+/**
+ * Finds dependency forms that cannot be represented by the static dependency
+ * IDs supplied by the backend build collector.
+ */
+function collectUnsupportedModuleDependencies(ast: Program): ModuleDependency[] {
+    const dependencies: ModuleDependency[] = [];
+
+    walkAst(ast, dependencies, {
+        ImportExpression(node, { state }) {
+            const specifier = getImportExpressionSpecifier(node);
+            if (shouldFailDynamicImport(specifier)) {
+                state.push({ specifier, kind: 'dynamic-import' });
+            }
+        },
+        CallExpression(node, { state }) {
+            if (isImportCallExpression(node)) {
+                const specifier = getImportCallSpecifier(node);
+                if (shouldFailDynamicImport(specifier)) {
+                    state.push({ specifier, kind: 'dynamic-import' });
+                }
+                return;
+            }
+
+            if (isLocalRequireCall(node)) {
+                state.push({
+                    specifier: getRequireSpecifier(node),
+                    kind: 'require',
+                });
+            }
+        },
+    });
+
+    return dependencies;
+}
+
+/**
+ * Dynamic package imports are skipped, but local or non-literal dynamic imports
+ * could hide app-local action-catalog calls and must fail closed.
+ */
+function shouldFailDynamicImport(specifier: string): boolean {
+    return specifier === 'non-literal dynamic import' || isLocalSpecifier(specifier);
+}
+
+/**
+ * Keeps reachability traversal scoped to app-local JavaScript/TypeScript source
+ * modules that the backend build collector can safely analyze.
+ */
+export function shouldTraverseCollectedModule(moduleId: string, buildRoot: string): boolean {
+    if (!BACKEND_CODE_EXTENSIONS.some((extension) => moduleId.endsWith(extension))) {
+        return false;
+    }
+
+    const relativePath = path.relative(path.resolve(buildRoot), moduleId);
+    if (relativePath.startsWith('..') || path.isAbsolute(relativePath)) {
+        return false;
+    }
+
+    return !relativePath.split(path.sep).some((segment) => PACKAGE_MANAGER_DIRS.has(segment));
+}
+
+/**
+ * Reads the static string specifier from an ESTree dynamic import expression.
+ */
+function getImportExpressionSpecifier(node: ImportExpression): string {
+    return getLiteralSpecifier(node.source, 'non-literal dynamic import');
+}
+
+/**
+ * Reads the static string specifier from Rollup's call-expression form for
+ * dynamic import.
+ */
+function getImportCallSpecifier(node: ImportCallExpression): string {
+    return getLiteralSpecifier(node.arguments[0], 'non-literal dynamic import');
+}
+
+/**
+ * Reads the static string specifier from a CommonJS require call.
+ */
+function getRequireSpecifier(node: SimpleCallExpression): string {
+    return getLiteralSpecifier(node.arguments[0], 'local require');
+}
+
+/**
+ * Returns a literal string value when available, otherwise a diagnostic label
+ * used in fail-closed error messages.
+ */
+function getLiteralSpecifier(node: unknown, fallback: string): string {
+    if (isStringLiteral(node)) {
+        return node.value;
+    }
+    return fallback;
+}
+
+/**
+ * Narrows Rollup's dynamic import call-expression representation.
+ */
+function isImportCallExpression(node: SimpleCallExpression): node is ImportCallExpression {
+    return (node.callee as { type: string }).type === 'Import';
+}
+
+/**
+ * Detects local CommonJS require calls. Package require calls are ignored
+ * because package modules are outside the app-local backend graph.
+ */
+function isLocalRequireCall(node: SimpleCallExpression): boolean {
+    if (node.callee.type !== 'Identifier' || node.callee.name !== 'require') {
+        return false;
+    }
+    const [source] = node.arguments;
+    return !source || !isStringLiteral(source) || isLocalSpecifier(source.value);
+}
+
+/**
+ * Returns whether an import specifier points at an app-local path.
+ */
+function isLocalSpecifier(specifier: string): boolean {
+    return specifier.startsWith('.') || specifier.startsWith('/');
+}
+
+/**
+ * Builds the common fail-closed error for graph shapes that could hide an
+ * action-catalog connection ID.
+ */
+export function unsupportedModuleGraphDependency(filePath: string, unsupported: string): Error {
+    return new Error(
+        `Unsupported local module graph for ${filePath}: ${unsupported} could hide an action-catalog connectionId.`,
+    );
+}