[APPS][Connections Part 8] Traverse backend module graph for connection IDs

packages/plugins/apps/src/backend/ast-parsing/extract-connection-ids-from-module-graph.test.ts

@@ -0,0 +1,353 @@
+// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2019-Present Datadog, Inc.
+
+import type { Program } from 'estree';
+import { parseAst } from 'rollup/parseAst';
+
+import { extractConnectionIdsFromModuleGraph } from './extract-connection-ids-from-module-graph';
+import { createParsedModuleRecord, type ParsedModuleRecord } from './module-graph';
+
+const buildRoot = '/project';
+const entryId = '/project/src/backend/actions.backend.js';
+
+function parse(code: string): Program {
+    return parseAst(code) as Program;
+}
+
+function createRecord(
+    id: string,
+    code: string,
+    staticDependencies: string[] = [],
+): ParsedModuleRecord {
+    const record = createParsedModuleRecord(id, buildRoot, parse(code), staticDependencies);
+    if (!record) {
+        throw new Error(`Expected ${id} to create a parsed module record`);
+    }
+    return record;
+}
+
+function extract(records: ParsedModuleRecord[]): string[] {
+    return extractConnectionIdsFromModuleGraph(
+        entryId,
+        new Map(records.map((record) => [record.id, record])),
+        buildRoot,
+    );
+}
+
+describe('Backend Functions - extractConnectionIdsFromModuleGraph', () => {
+    test('Should return null when creating records for modules outside the backend graph', () => {
+        expect(
+            createParsedModuleRecord(
+                '/project/node_modules/package/index.js',
+                buildRoot,
+                parse('export const value = true;'),
+            ),
+        ).toBeNull();
+    });
+
+    test('Should extract inline connection IDs from statically reachable helper modules', () => {
+        const helperId = '/project/src/backend/helpers/http.js';
+        const entry = createRecord(
+            entryId,
+            `
+                import { getEcho } from './helpers/http.js';
+
+                export function run() {
+                    return getEcho();
+                }
+            `,
+            [helperId],
+        );
+        const helper = createRecord(
+            helperId,
+            `
+                import { request } from '@datadog/action-catalog/http/http';
+
+                export function getEcho() {
+                    return request({ connectionId: 'conn-helper', inputs: {} });
+                }
+            `,
+        );
+
+        expect(extract([entry, helper])).toEqual(['conn-helper']);
+    });
+
+    test('Should resolve same-module connection ID values inside reachable helpers', () => {
+        const helperId = '/project/src/backend/helpers/http.js';
+        const entry = createRecord(
+            entryId,
+            `
+                import { getEcho } from './helpers/http.js';
+
+                export function run() {
+                    return getEcho();
+                }
+            `,
+            [helperId],
+        );
+        const helper = createRecord(
+            helperId,
+            `
+                import { request } from '@datadog/action-catalog/http/http';
+
+                const HTTP_CONNECTION_ID = 'conn-const';
+                const CONNECTIONS = { HTTP: { PROD: 'conn-object' } };
+
+                export function getEcho() {
+                    request({ connectionId: HTTP_CONNECTION_ID, inputs: {} });
+                    request({ connectionId: CONNECTIONS.HTTP.PROD, inputs: {} });
+                }
+            `,
+        );
+
+        expect(extract([entry, helper])).toEqual(['conn-const', 'conn-object']);
+    });
+
+    test('Should traverse named re-exports and export star declarations', () => {
+        const barrelId = '/project/src/backend/helpers/index.js';
+        const namedId = '/project/src/backend/helpers/named.js';
+        const starId = '/project/src/backend/helpers/star.js';
+        const entry = createRecord(
+            entryId,
+            `
+                import './helpers/index.js';
+
+                export function run() {}
+            `,
+            [barrelId],
+        );
+        const barrel = createRecord(
+            barrelId,
+            `
+                export { getNamed } from './named.js';
+                export * from './star.js';
+            `,
+            [namedId, starId],
+        );
+        const named = createRecord(
+            namedId,
+            `
+                import { request } from '@datadog/action-catalog/http/http';
+                export function getNamed() {
+                    return request({ connectionId: 'conn-named', inputs: {} });
+                }
+            `,
+        );
+        const star = createRecord(
+            starId,
+            `
+                import { request } from '@datadog/action-catalog/http/http';
+                export function getStar() {
+                    return request({ connectionId: 'conn-star', inputs: {} });
+                }
+            `,
+        );
+
+        expect(extract([entry, barrel, named, star])).toEqual(['conn-named', 'conn-star']);
+    });
+
+    test('Should ignore package imports while traversing collected records', () => {
+        const entry = createRecord(
+            entryId,
+            `
+                import { helper } from 'some-package';
+
+                export function run() {}
+            `,
+            ['/project/node_modules/some-package/index.js'],
+        );
+
+        expect(extract([entry])).toEqual([]);
+    });
+
+    test.each([
+        { description: 'outside buildRoot', resolvedId: '/external/helper.js' },
+        { description: 'virtual modules', resolvedId: '\0virtual-helper.js' },
+        { description: 'package modules', resolvedId: '/project/node_modules/package/index.js' },
+        {
+            description: 'Yarn package cache modules',
+            resolvedId: '/project/.yarn/cache/package/index.js',
+        },
+        { description: 'non-JavaScript files', resolvedId: '/project/src/backend/data.json' },
+    ])('Should skip $description', ({ resolvedId }) => {
+        const entry = createRecord(
+            entryId,
+            `
+                import './helper.js';
+
+                export function run() {}
+            `,
+            [resolvedId],
+        );
+
+        expect(extract([entry])).toEqual([]);
+    });
+
+    test.each([
+        { folder: 'dist', connectionId: 'conn-dist' },
+        { folder: 'build', connectionId: 'conn-build' },
+        { folder: '.vite', connectionId: 'conn-vite' },
+    ])('Should traverse supported app-local folder name $folder', ({ folder, connectionId }) => {
+        const helperId = `/project/${folder}/helper.js`;
+        const entry = createRecord(
+            entryId,
+            `
+                import '../${folder}/helper.js';
+
+                export function run() {}
+            `,
+            [helperId],
+        );
+        const helper = createRecord(
+            helperId,
+            `
+                import { request } from '@datadog/action-catalog/http/http';
+
+                request({ connectionId: '${connectionId}', inputs: {} });
+            `,
+        );
+
+        expect(extract([entry, helper])).toEqual([connectionId]);
+    });
+
+    test('Should protect against local graph cycles', () => {
+        const aId = '/project/src/backend/a.js';
+        const bId = '/project/src/backend/b.js';
+        const entry = createRecord(
+            entryId,
+            `
+                import './a.js';
+
+                export function run() {}
+            `,
+            [aId],
+        );
+        const a = createRecord(
+            aId,
+            `
+                import './b.js';
+                import { request } from '@datadog/action-catalog/http/http';
+                request({ connectionId: 'conn-a', inputs: {} });
+            `,
+            [bId],
+        );
+        const b = createRecord(
+            bId,
+            `
+                import './a.js';
+                import { request } from '@datadog/action-catalog/http/http';
+                request({ connectionId: 'conn-b', inputs: {} });
+            `,
+            [aId],
+        );
+
+        expect(extract([entry, a, b])).toEqual(['conn-a', 'conn-b']);
+    });
+
+    test.each([
+        {
+            description: 'dynamic local imports',
+            code: "import('./helper.js');",
+            message: 'dynamic-import ./helper.js',
+        },
+        {
+            description: 'non-literal dynamic imports',
+            code: 'import(helperPath);',
+            message: 'dynamic-import non-literal dynamic import',
+        },
+        {
+            description: 'local require calls',
+            code: "require('./helper.js');",
+            message: 'require ./helper.js',
+        },
+    ])('Should fail closed for $description', ({ code, message }) => {
+        const entry = createRecord(
+            entryId,
+            `
+                ${code}
+
+                export function run() {}
+            `,
+        );
+
+        expect(() => extract([entry])).toThrow(message);
+    });
+
+    test('Should fail closed for uncollected local static imports', () => {
+        const missingId = '/project/src/backend/missing.js';
+        const entry = createRecord(
+            entryId,
+            `
+                import './missing.js';
+
+                export function run() {}
+            `,
+            [missingId],
+        );
+
+        expect(() => extract([entry])).toThrow(`uncollected local import ${missingId}`);
+    });
+
+    test('Should keep imported connection ID values unsupported in reachable helpers', () => {
+        const helperId = '/project/src/backend/helpers/http.js';
+        const idsId = '/project/src/backend/helpers/ids.js';
+        const entry = createRecord(
+            entryId,
+            `
+                import { getEcho } from './helpers/http.js';
+
+                export function run() {
+                    return getEcho();
+                }
+            `,
+            [helperId],
+        );
+        const helper = createRecord(
+            helperId,
+            `
+                import { request } from '@datadog/action-catalog/http/http';
+                import { HTTP_CONNECTION_ID } from './ids.js';
+
+                export function getEcho() {
+                    return request({ connectionId: HTTP_CONNECTION_ID, inputs: {} });
+                }
+            `,
+            [idsId],
+        );
+
+        expect(() => extract([entry, helper])).toThrow(
+            'imported connectionId binding HTTP_CONNECTION_ID',
+        );
+    });
+
+    test('Should read transformed local TypeScript helpers as collected records', () => {
+        const helperId = '/project/src/backend/helpers/http.ts';
+        const entry = createRecord(
+            entryId,
+            `
+                import { getEcho } from './helpers/http';
+
+                export function run() {
+                    return getEcho();
+                }
+            `,
+            [helperId],
+        );
+        const helper = createRecord(
+            helperId,
+            `
+                import { request } from '@datadog/action-catalog/http/http';
+
+                const HTTP_CONNECTION_ID = 'conn-ts';
+
+                export function getEcho() {
+                    request({ connectionId: HTTP_CONNECTION_ID, inputs: {} });
+                    return { ok: true };
+                }
+            `,
+        );
+
+        expect(extract([entry, helper])).toEqual(['conn-ts']);
+    });
+});

packages/plugins/apps/src/backend/ast-parsing/extract-connection-ids-from-module-graph.ts

@@ -0,0 +1,31 @@
+// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2019-Present Datadog, Inc.
+
+import { extractConnectionIds } from './extract-connection-ids';
+import type { ParsedModuleRecord } from './module-graph';
+import { walkModuleGraph } from './walk-module-graph';
+
+/**
+ * Extracts the conservative backend-file connection ID union from module records
+ * collected while the backend bundler walked the real execution graph.
+ */
+export function extractConnectionIdsFromModuleGraph(
+    entryId: string,
+    modules: ReadonlyMap<string, ParsedModuleRecord>,
+    buildRoot: string,
+): string[] {
+    const connectionIds = new Set<string>();
+
+    // Walk the already-parsed records from this backend entry's build. The
+    // extraction cost is linear in reachable app-local modules, without
+    // reparsing source files here.
+    walkModuleGraph(entryId, modules, buildRoot, ({ moduleId, record }) => {
+        const moduleConnectionIds = extractConnectionIds(record.ast, moduleId);
+        for (const connectionId of moduleConnectionIds) {
+            connectionIds.add(connectionId);
+        }
+    });
+
+    return [...connectionIds].sort();
+}