Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,10 @@ jobs:
# Use production build for faster startup
NODE_ENV: production

- name: Run component tests
working-directory: ./ui
run: pnpm run test:ct

- name: Upload Playwright report
uses: actions/upload-artifact@v4
if: failure()
Expand Down
9 changes: 8 additions & 1 deletion server/src/agent_control_server/endpoints/controls.py
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,11 @@ def _serialize_control_definition(control_def: ControlDefinition) -> dict[str, o
async def _validate_control_definition(
control_def: ControlDefinition, db: AsyncSession
) -> None:
"""Validate evaluator config for a control definition."""
"""Validate evaluator config for definitions referencing known global evaluators.

Agent-scoped evaluators must exist on the referenced agent. Builtin and external
names that are not loaded in this process are accepted without config checks.
"""
available_evaluators = list_evaluators()
agent_data_by_name: dict[str, AgentData] = {}
for field_prefix, leaf in _iter_condition_leaves(control_def.condition):
Expand Down Expand Up @@ -212,6 +216,9 @@ async def _validate_control_definition(

evaluator_cls = available_evaluators.get(parsed.name)
if evaluator_cls is None:
# Global (builtin / external) evaluators may be absent from this runtime
# (optional packages, forward compatibility). Store the definition without
# config validation; evaluation will fail later if the evaluator is missing.
continue

try:
Expand Down
29 changes: 29 additions & 0 deletions server/tests/test_controls_validation.py
Original file line number Diff line number Diff line change
Expand Up @@ -318,3 +318,32 @@ def test_validation_nested_agent_scoped_evaluator_error_uses_bracketed_field_pat
and err.get("code") == "evaluator_not_found"
for err in body.get("errors", [])
)


def test_validation_standalone_evaluator_error_uses_bracketed_field_path(
client: TestClient,
):
"""Nested standalone (global) evaluator config errors use bracketed leaf paths."""
control_id = create_control(client)
payload = deepcopy(VALID_CONTROL_PAYLOAD)
payload["condition"] = {
"or": [
{
"selector": {"path": "input"},
"evaluator": {
"name": "regex",
"config": {},
},
}
]
}

resp = client.put(f"/api/v1/controls/{control_id}/data", json={"data": payload})

assert resp.status_code == 422
body = resp.json()
assert body["error_code"] == "VALIDATION_ERROR"
assert any(
err.get("field", "").startswith("data.condition.or[0].evaluator")
for err in body.get("errors", [])
)
2 changes: 2 additions & 0 deletions ui/.gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -42,3 +42,5 @@ next-env.d.ts

CLAUDE.md
.claude

playwright/.cache/
20 changes: 19 additions & 1 deletion ui/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
"typecheck": "tsc --noEmit",
"fetch-api-types": "openapi-typescript http://localhost:8000/openapi.json -o src/core/api/generated/api-types.ts",
"test:integration": "playwright test",
"test:ct": "playwright test -c playwright-ct.config.ts",
"test:ct:ui": "playwright test -c playwright-ct.config.ts --ui",
"test:integration:ui": "playwright test --ui",
"test:integration:headed": "playwright test --headed",
"test:integration:debug": "playwright test --debug",
Expand All @@ -31,6 +33,8 @@
"typecheck": "Run TypeScript checks (no emit)",
"fetch-api-types": "Regenerate API types from the server OpenAPI schema at localhost:8000",
"test:integration": "Run Playwright integration tests",
"test:ct": "Run Playwright component tests (JsonEditor, etc.)",
"test:ct:ui": "Run component tests in interactive UI mode",
"test:integration:ui": "Run Playwright integration tests in interactive UI mode",
"test:integration:headed": "Run Playwright integration tests in headed browser mode",
"test:integration:debug": "Run Playwright integration tests in debug mode",
Expand All @@ -39,7 +43,14 @@
"prettify:check": "Check formatting with Prettier without changing files"
},
"dependencies": {
"@codemirror/autocomplete": "^6.20.1",
"@codemirror/lang-json": "^6.0.2",
"@codemirror/language": "^6.12.3",
"@codemirror/lint": "^6.9.5",
"@codemirror/state": "^6.6.0",
"@codemirror/view": "^6.40.0",
"@emotion/is-prop-valid": "^1.4.0",
"@lezer/highlight": "^1.2.3",
"@mantine/charts": "^7.17.8",
"@mantine/code-highlight": "7.17.5",
"@mantine/core": "7.17.5",
Expand All @@ -56,6 +67,10 @@
"@tanstack/react-query": "5.74.4",
"@tanstack/react-query-devtools": "5.72.2",
"@tanstack/react-table": "8.20.5",
"@uiw/codemirror-extensions-basic-setup": "^4.25.9",
"@uiw/codemirror-themes": "^4.25.9",
"@uiw/codemirror-themes-all": "4.25.9",
"@uiw/react-codemirror": "^4.25.9",
"axios": "1.12.0",
"classix": "2.2.0",
"date-fns": "4.1.0",
Expand All @@ -71,11 +86,13 @@
},
"devDependencies": {
"@eslint/eslintrc": "^3.3.3",
"@playwright/experimental-ct-react": "1.57.0",
"@playwright/test": "^1.57.0",
"@tailwindcss/postcss": "^4",
"@types/node": "^20",
"@types/react": "^19",
"@types/react-dom": "^19",
"@vitejs/plugin-react": "^4.7.0",
"eslint": "^9",
"eslint-config-next": "16.1.1",
"eslint-plugin-simple-import-sort": "^12.1.1",
Expand All @@ -87,6 +104,7 @@
"prettier": "^3.4.2",
"tailwindcss": "^4",
"typescript": "^5",
"typescript-eslint": "^8.32.1"
"typescript-eslint": "^8.32.1",
"vite": "^6.4.1"
}
}
35 changes: 35 additions & 0 deletions ui/playwright-ct.config.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
import path from 'node:path';

import { defineConfig, devices } from '@playwright/experimental-ct-react';

/**
* Component tests: mount React in-browser via Vite (no Next.js server).
* @see https://playwright.dev/docs/test-components
*/
export default defineConfig({
testDir: './tests/ct',
fullyParallel: true,
forbidOnly: !!process.env.CI,
retries: process.env.CI ? 2 : 0,
workers: process.env.CI ? 2 : undefined,
reporter: [['html', { open: 'never' }], ['list']],

use: {
trace: 'on-first-retry',
screenshot: 'only-on-failure',
ctViteConfig: {
resolve: {
alias: {
'@': path.resolve(__dirname, './src'),
},
},
},
},

projects: [
{
name: 'chromium',
use: { ...devices['Desktop Chrome'] },
},
],
});
2 changes: 2 additions & 0 deletions ui/playwright.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import { defineConfig, devices } from '@playwright/test';
*/
export default defineConfig({
testDir: './tests',
/* Component tests use playwright-ct.config.ts (Vite + mount). */
testIgnore: '**/ct/**',
/* Run tests in files in parallel */
fullyParallel: true,
/* Fail the build on CI if you accidentally left test.only in the source code. */
Expand Down
12 changes: 12 additions & 0 deletions ui/playwright/index.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Playwright CT</title>
</head>
<body>
<div id="root"></div>
<script type="module" src="./index.tsx"></script>
</body>
</html>
28 changes: 28 additions & 0 deletions ui/playwright/index.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
import '@mantine/charts/styles.css';
import '@mantine/code-highlight/styles.css';
import '@mantine/core/styles.css';
import '@mantine/dates/styles.css';
import '@mantine/notifications/styles.css';
import '@rungalileo/icons/styles.css';
import '@rungalileo/jupiter-ds/styles.css';
import '@/styles/globals.css';

import { MantineProvider } from '@mantine/core';
import { DatesProvider } from '@mantine/dates';
import { ModalsProvider } from '@mantine/modals';
import { beforeMount } from '@playwright/experimental-ct-react/hooks';
import { JupiterThemeProvider } from '@rungalileo/jupiter-ds';

import { appTheme } from '@/theme';

beforeMount(async ({ App }) => (
<MantineProvider theme={appTheme} defaultColorScheme="light">
<DatesProvider settings={{ firstDayOfWeek: 0 }}>
<JupiterThemeProvider>
<ModalsProvider>
<App />
</ModalsProvider>
</JupiterThemeProvider>
</DatesProvider>
</MantineProvider>
));
Loading
Loading