diff --git a/project-demos/ivy-ask-statistics/.dockerignore b/project-demos/ivy-ask-statistics/.dockerignore
new file mode 100644
index 00000000..2f32bfe4
--- /dev/null
+++ b/project-demos/ivy-ask-statistics/.dockerignore
@@ -0,0 +1,25 @@
+**/.dockerignore
+**/.env
+**/.git
+**/.gitignore
+**/.project
+**/.settings
+**/.toolstarget
+**/.vs
+**/.vscode
+**/.idea
+**/*.*proj.user
+**/*.dbmdl
+**/*.jfm
+**/azds.yaml
+**/bin
+**/charts
+**/docker-compose*
+**/Dockerfile*
+**/node_modules
+**/npm-debug.log
+**/obj
+**/secrets.dev.yaml
+**/values.dev.yaml
+LICENSE
+README.md
diff --git a/project-demos/ivy-ask-statistics/Apps/DashboardApp.cs b/project-demos/ivy-ask-statistics/Apps/DashboardApp.cs
new file mode 100644
index 00000000..0f50c4e9
--- /dev/null
+++ b/project-demos/ivy-ask-statistics/Apps/DashboardApp.cs
@@ -0,0 +1,817 @@
+namespace IvyAskStatistics.Apps;
+
+[App(icon: Icons.LayoutDashboard, title: "Dashboard")]
+public class DashboardApp : ViewBase
+{
+ ///
+ /// Survives tab switches: state is recreated when the view remounts,
+ /// but we still need the last successful payload so a failed refetch does not wipe the UI.
+ /// Use a unique string query key (not shared 0 with other apps) so the server cache is not clobbered.
+ ///
+ private static readonly Dictionary s_lastDashboardByKey = new(StringComparer.Ordinal);
+
+ public override object? Build()
+ {
+ var factory = UseService();
+ var client = UseService();
+ var navigation = Context.UseNavigation();
+ var selectedRunId = UseState(null);
+ var runDialogOpen = UseState(false);
+ var editSheetOpen = UseState(false);
+ var editQuestionId = UseState(Guid.Empty);
+ var editPreviewResultId = UseState(null);
+ var envOverride = UseState("production");
+ var dashboardFocusVersion = UseState(null);
+ var versionSheetOpen = UseState(false);
+
+ var dashQuery = UseQuery(
+ key: DashboardQueryKey(dashboardFocusVersion.Value),
+ fetcher: async (key, ct) =>
+ {
+ var focusVersion = ParseDashboardFocusFromQueryKey(key);
+ try
+ {
+ var r = await LoadDashboardPageAsync(factory, focusVersion, ct);
+ s_lastDashboardByKey[key] = r;
+ return r;
+ }
+ catch (OperationCanceledException)
+ {
+ return s_lastDashboardByKey.TryGetValue(key, out var prev) ? prev : null;
+ }
+ catch (Exception ex) when (!ct.IsCancellationRequested)
+ {
+ client.Toast($"Could not load dashboard: {ex.Message}");
+ return s_lastDashboardByKey.TryGetValue(key, out var prev) ? prev : null;
+ }
+ },
+ options: new QueryOptions { KeepPrevious = false },
+ tags: ["dashboard-stats"]);
+
+ var dashQueryKey = DashboardQueryKey(dashboardFocusVersion.Value);
+
+ // Prefer live query value; fall back to last success for this query key when remounting or on transient errors.
+ var page = dashQuery.Value ?? (s_lastDashboardByKey.TryGetValue(dashQueryKey, out var cached) ? cached : null);
+
+ if (dashQuery.Loading && page == null)
+ return TabLoadingSkeletons.Dashboard();
+
+ if (page == null)
+ return Layout.Vertical().Height(Size.Full()).AlignContent(Align.Center)
+ | new Icon(Icons.LayoutDashboard)
+ | Text.H3("No statistics yet")
+ | Text.Block("No completed test runs found. Run a test to see dashboard statistics.")
+ .Muted()
+ | new Button("Run Tests", onClick: _ => navigation.Navigate(typeof(RunApp)))
+ .Primary()
+ .Icon(Icons.Play);
+ var versionCompare = page.VersionCompare;
+
+ // Resolve data for each env from what the loader returned.
+ var prodData = string.Equals(page.PrimaryEnvironment, "production", StringComparison.OrdinalIgnoreCase)
+ ? page.Detail : page.PeerDetail;
+ var stgData = string.Equals(page.PrimaryEnvironment, "staging", StringComparison.OrdinalIgnoreCase)
+ ? page.Detail : page.PeerDetail;
+ var hasStagingData = stgData != null;
+ var hasProductionData = prodData != null;
+
+ // Respect the user's toggle; fall back gracefully when an env has no data.
+ var showEnv = envOverride.Value;
+ if (showEnv == "staging" && !hasStagingData) showEnv = "production";
+ if (showEnv == "production" && !hasProductionData) showEnv = "staging";
+
+ DashboardData data;
+ DashboardData? peer;
+ string envPrimary;
+ if (showEnv == "staging" && stgData != null)
+ {
+ data = stgData;
+ peer = prodData;
+ envPrimary = "Staging";
+ }
+ else
+ {
+ data = prodData ?? page.Detail;
+ peer = stgData;
+ envPrimary = "Production";
+ }
+ var hasPeerCompare = peer != null;
+
+ // Remount KPIs + charts when the selected Ivy version or env slice changes. Keep this stable (no live metrics)
+ // so we do not get a new key every build — that would remount widgets and retrigger CSS animations constantly.
+ var dashboardVisualKey = $"{dashQueryKey}|{envPrimary}";
+
+ // ── Level 1: KPIs (IvyInsights-style headline + delta vs other env or vs previous version) ──
+ var rateStr = $"{data.AnswerRate:F1}%";
+ object rateDelta = hasPeerCompare
+ ? FormatDeltaWithTrend(data.AnswerRate - peer!.AnswerRate, "%", higherIsBetter: true)
+ : data.PrevAnswerRate.HasValue
+ ? FormatDeltaWithTrend(data.AnswerRate - data.PrevAnswerRate.Value, "%", higherIsBetter: true)
+ : Text.Muted("no baseline");
+
+ // NBSP keeps "181" and "ms" on one line; narrow / Fit columns otherwise wrap between tokens.
+ var avgMsStr = $"{data.AvgMs}\u00A0ms";
+ object avgMsDelta = hasPeerCompare
+ ? FormatDeltaWithTrend(data.AvgMs - peer!.AvgMs, "ms", higherIsBetter: false)
+ : data.PrevAvgMs.HasValue
+ ? FormatDeltaWithTrend(data.AvgMs - data.PrevAvgMs.Value, "ms", higherIsBetter: false)
+ : Text.Muted("no baseline");
+
+ var failedCount = data.NoAnswer + data.Errors;
+ var peerFailed = hasPeerCompare ? peer!.NoAnswer + peer.Errors : (int?)null;
+ object failedDelta = hasPeerCompare && peerFailed.HasValue
+ ? FormatDeltaWithTrend(failedCount - peerFailed.Value, "", higherIsBetter: false, countMode: true)
+ : new Empty();
+
+ var runVersion = string.IsNullOrWhiteSpace(page.IvyVersion) ? "—" : page.IvyVersion.Trim();
+
+ var kpiRow = Layout.Grid().Columns(5).Height(Size.Fit()).Key(dashboardVisualKey + "|kpi")
+ | new Card(
+ Layout.Vertical().AlignContent(Align.Center)
+ | (Layout.Horizontal().AlignContent(Align.Center).Gap(1)
+ | Text.H2(rateStr).Bold()
+ | rateDelta)
+ ).Title("Answer success").Icon(Icons.CircleCheck)
+ | new Card(
+ Layout.Vertical().AlignContent(Align.Center)
+ | (Layout.Horizontal().AlignContent(Align.Center).Gap(1)
+ | Text.H2(avgMsStr).Bold()
+ | avgMsDelta)
+ ).Title("Avg latency").Icon(Icons.Timer)
+ | new Card(
+ Layout.Vertical().AlignContent(Align.Center)
+ | (Layout.Horizontal().AlignContent(Align.Center).Gap(1)
+ | Text.H2(failedCount.ToString("N0")).Bold()
+ | failedDelta)
+ ).Title("No answer + errors").Icon(Icons.CircleX)
+ | new Card(
+ Layout.Vertical().AlignContent(Align.Center)
+ | Text.H2(data.WorstWidgets.Count > 0 ? data.WorstWidgets[0].Widget : "—").Bold()
+ ).Title("Weakest widget").Icon(Icons.Ban)
+ | new Card(
+ Layout.Vertical().AlignContent(Align.Center)
+ | Text.H2(runVersion).Bold()
+ ).Title("Ivy version").Icon(Icons.Tag)
+ .OnClick(versionCompare.Count > 0
+ ? _ => versionSheetOpen.Set(true)
+ : _ => { });
+
+ // ── Production vs staging by Ivy version ──
+ object versionChartsRow;
+ if (versionCompare.Count >= 1)
+ {
+ var rateByVersion = versionCompare.ToBarChart()
+ .Dimension("Version", x => x.Version)
+ .Measure("Production %", x => x.Sum(f => f.ProductionAnswerRate))
+ .Measure("Staging %", x => x.Sum(f => f.StagingAnswerRate));
+
+ var latencyByVersion = versionCompare.ToBarChart()
+ .Dimension("Version", x => x.Version)
+ .Measure("Production ms", x => x.Sum(f => f.ProductionAvgMs))
+ .Measure("Staging ms", x => x.Sum(f => f.StagingAvgMs));
+
+ var outcomesByVersion = versionCompare.ToBarChart()
+ .Dimension("Version", x => x.Version)
+ .Measure("Prod answered", x => x.Sum(f => f.ProductionAnswered))
+ .Measure("Stg answered", x => x.Sum(f => f.StagingAnswered))
+ .Measure("Prod no answer", x => x.Sum(f => f.ProductionNoAnswer))
+ .Measure("Stg no answer", x => x.Sum(f => f.StagingNoAnswer))
+ .Measure("Prod error", x => x.Sum(f => f.ProductionErrors))
+ .Measure("Stg error", x => x.Sum(f => f.StagingErrors));
+
+ versionChartsRow = Layout.Grid().Columns(3).Height(Size.Fit()).Key(dashboardVisualKey + "|vercmp")
+ | new Card(rateByVersion).Title("Success rate · production vs staging").Height(Size.Units(70))
+ | new Card(latencyByVersion).Title("Avg response · production vs staging").Height(Size.Units(70))
+ | new Card(outcomesByVersion).Title("Outcomes · production vs staging").Height(Size.Units(70));
+ }
+ else
+ {
+ versionChartsRow = Layout.Vertical();
+ }
+
+ var worstChart = data.WorstWidgets.ToBarChart()
+ .Dimension("Widget", x => x.Widget)
+ .Measure("Answer rate %", x => x.Sum(f => f.AnswerRate));
+
+ var latencyByWidgetChart = data.WorstWidgets.ToBarChart()
+ .Dimension("Widget", x => x.Widget)
+ .Measure("Avg ms", x => x.Sum(f => (double)f.AvgMs))
+ .Measure("Max ms", x => x.Sum(f => (double)f.MaxMs));
+
+ var resultDistribution = new[]
+ {
+ new { Label = "Answered", Count = data.Answered },
+ new { Label = "No answer", Count = data.NoAnswer },
+ new { Label = "Error", Count = data.Errors }
+ }.Where(x => x.Count > 0).ToList();
+
+ var pieChart = resultDistribution.ToPieChart(
+ dimension: x => x.Label,
+ measure: x => x.Sum(f => f.Count),
+ PieChartStyles.Dashboard,
+ new PieChartTotal(data.Total.ToString("N0"), "Total"));
+
+ var difficultyChart = data.DifficultyBreakdown.ToBarChart()
+ .Dimension("Difficulty", x => x.Difficulty)
+ .Measure("Answered", x => x.Sum(f => f.Answered))
+ .Measure("No answer", x => x.Sum(f => f.NoAnswer))
+ .Measure("Error", x => x.Sum(f => f.Errors));
+
+ // Row 1: 3 charts (prod vs staging by version — always both envs)
+ // Row 2: 4 charts (per-selected-env diagnostics)
+ var chartsRow = Layout.Grid().Columns(4).Height(Size.Fit()).Key(dashboardVisualKey + "|detail-charts")
+ | new Card(worstChart).Title($"Worst widgets — rate % ({envPrimary})").Height(Size.Units(70))
+ | new Card(latencyByWidgetChart).Title($"Latency by widget — avg / max ({envPrimary})").Height(Size.Units(70))
+ | new Card(difficultyChart).Title($"Results by difficulty ({envPrimary})").Height(Size.Units(70))
+ | new Card(pieChart).Title($"Result mix ({envPrimary})").Height(Size.Units(70));
+
+ // ── Test runs table (full width) ──
+ var runsTable = page.AllRuns.AsQueryable()
+ .ToDataTable(r => r.Id)
+ .Height(Size.Units(120))
+ .Key($"all-test-runs|{dashQueryKey}")
+ .Header(r => r.IvyVersion, "Ivy version")
+ .Header(r => r.Environment, "Environment")
+ .Header(r => r.DifficultyFilter, "Difficulty")
+ .Header(r => r.TotalQuestions, "Total")
+ .Header(r => r.SuccessCount, "Answered")
+ .Header(r => r.NoAnswerCount, "No answer")
+ .Header(r => r.ErrorCount, "Errors")
+ .Header(r => r.AnswerRate, "Rate %")
+ .Header(r => r.AvgMs, "Avg ms")
+ .Header(r => r.StartedAt, "Started")
+ .Header(r => r.CompletedAt, "Completed")
+ .Width(r => r.IvyVersion, Size.Px(120))
+ .Width(r => r.Environment, Size.Px(100))
+ .Width(r => r.DifficultyFilter, Size.Px(80))
+ .Width(r => r.TotalQuestions, Size.Px(60))
+ .Width(r => r.SuccessCount, Size.Px(80))
+ .Width(r => r.NoAnswerCount, Size.Px(80))
+ .Width(r => r.ErrorCount, Size.Px(60))
+ .Width(r => r.AnswerRate, Size.Px(70))
+ .Width(r => r.AvgMs, Size.Px(70))
+ .Width(r => r.StartedAt, Size.Px(160))
+ .Width(r => r.CompletedAt, Size.Px(160))
+ .Hidden(r => r.Id)
+ .RowActions(
+ MenuItem.Default(Icons.Eye, "view").Label("View results").Tag("view"))
+ .OnRowAction(e =>
+ {
+ var args = e.Value;
+ if (args?.Tag?.ToString() == "view" && Guid.TryParse(args?.Id?.ToString(), out var id))
+ {
+ selectedRunId.Set(id);
+ runDialogOpen.Set(true);
+ }
+ return ValueTask.CompletedTask;
+ })
+ .Config(c =>
+ {
+ c.AllowSorting = true;
+ c.AllowFiltering = true;
+ c.ShowSearch = true;
+ c.ShowIndexColumn = false;
+ });
+ var tableRuns = Layout.Vertical() | new Card(runsTable).Title("Test runs");
+
+ var mainLayout = Layout.Vertical().Height(Size.Full())
+ | kpiRow
+ | versionChartsRow
+ | chartsRow
+ | tableRuns ;
+
+ return new Fragment(
+ mainLayout,
+ versionSheetOpen.Value && versionCompare.Count > 0
+ ? new DashboardVersionPickerSheet(
+ versionSheetOpen,
+ dashboardFocusVersion,
+ versionCompare,
+ (page.IvyVersion ?? "").Trim())
+ : new Empty(),
+ runDialogOpen.Value && selectedRunId.Value.HasValue
+ ? new TestRunResultsDialog(
+ runDialogOpen,
+ selectedRunId.Value.Value,
+ editSheetOpen,
+ editQuestionId,
+ editPreviewResultId)
+ : new Empty(),
+ editSheetOpen.Value
+ ? new QuestionEditSheet(editSheetOpen, editQuestionId.Value, editPreviewResultId)
+ : new Empty());
+ }
+
+ private const char DashboardQueryKeySep = '\u001f';
+
+ private static string DashboardQueryKey(string? ivyVersionFocus) =>
+ "dashboard-stats-page" + DashboardQueryKeySep + (ivyVersionFocus ?? "");
+
+ private static string? ParseDashboardFocusFromQueryKey(string key)
+ {
+ var i = key.IndexOf(DashboardQueryKeySep);
+ if (i < 0 || i >= key.Length - 1)
+ return null;
+ var tail = key[(i + 1)..].Trim();
+ return string.IsNullOrEmpty(tail) ? null : tail;
+ }
+
+ private static string CapitalizeEnv(string env) =>
+ env.Equals("staging", StringComparison.OrdinalIgnoreCase) ? "Staging" : "Production";
+
+ private static string NormalizeEnvironment(string? environment)
+ {
+ var e = (environment ?? "").Trim().ToLowerInvariant();
+ return e == "staging" ? "staging" : "production";
+ }
+
+ /// Trend icon + colored delta (same idea as IvyInsights KPI cards).
+ private static object FormatDeltaWithTrend(double delta, string unit, bool higherIsBetter, bool countMode = false)
+ {
+ if (countMode)
+ {
+ if (delta == 0) return Text.Muted("—");
+ }
+ else if (unit == "%")
+ {
+ if (Math.Abs(delta) < 0.05) return Text.Muted("—");
+ }
+ else if (Math.Abs(delta) < 1) return Text.Muted("—");
+
+ var sign = delta > 0 ? "+" : "";
+ var label = countMode
+ ? $"{sign}{(int)delta}"
+ : unit == "%"
+ ? $"{sign}{delta:F1}{unit}"
+ : $"{sign}{(int)delta} {unit}";
+ var isGood = higherIsBetter ? delta > 0 : delta < 0;
+ var icon = isGood ? Icons.TrendingUp : Icons.TrendingDown;
+ var color = isGood ? Colors.Success : Colors.Destructive;
+ return Layout.Horizontal().Gap(1).AlignContent(Align.Center)
+ | new Icon(icon).Color(color)
+ | Text.H3(label).Color(color);
+ }
+
+ private static async Task LoadDashboardPageAsync(
+ AppDbContextFactory factory, string? ivyVersionFocus, CancellationToken ct)
+ {
+ await using var ctx = factory.CreateDbContext();
+
+ var runIdsWithData = await ctx.TestResults.AsNoTracking()
+ .Select(r => r.TestRunId)
+ .Distinct()
+ .ToListAsync(ct);
+
+ if (runIdsWithData.Count == 0) return null;
+
+ var runs = await ctx.TestRuns.AsNoTracking()
+ .Where(r => r.CompletedAt != null && runIdsWithData.Contains(r.Id))
+ .OrderByDescending(r => r.StartedAt)
+ .ToListAsync(ct);
+
+ if (runs.Count == 0) return null;
+
+ var latestProdByVersion = new Dictionary(StringComparer.OrdinalIgnoreCase);
+ var latestStagByVersion = new Dictionary(StringComparer.OrdinalIgnoreCase);
+ foreach (var r in runs)
+ {
+ var v = (r.IvyVersion ?? "").Trim();
+ if (string.IsNullOrEmpty(v)) continue;
+ var dict = NormalizeEnvironment(r.Environment) == "staging" ? latestStagByVersion : latestProdByVersion;
+ if (!dict.ContainsKey(v))
+ dict[v] = r;
+ }
+
+ var latestProd = runs.FirstOrDefault(r => NormalizeEnvironment(r.Environment) == "production");
+ var latestStag = runs.FirstOrDefault(r => NormalizeEnvironment(r.Environment) == "staging");
+
+ TestRunEntity primaryRun;
+ string primaryEnv;
+ var focus = (ivyVersionFocus ?? "").Trim();
+ if (string.IsNullOrEmpty(focus))
+ {
+ primaryRun = latestProd ?? latestStag ?? runs[0];
+ primaryEnv = NormalizeEnvironment(primaryRun.Environment);
+ }
+ else
+ {
+ latestProdByVersion.TryGetValue(focus, out var prFocus);
+ latestStagByVersion.TryGetValue(focus, out var srFocus);
+ if (prFocus != null)
+ {
+ primaryRun = prFocus;
+ primaryEnv = "production";
+ }
+ else if (srFocus != null)
+ {
+ primaryRun = srFocus;
+ primaryEnv = "staging";
+ }
+ else
+ {
+ primaryRun = latestProd ?? latestStag ?? runs[0];
+ primaryEnv = NormalizeEnvironment(primaryRun.Environment);
+ }
+ }
+
+ var allVersions = latestProdByVersion.Keys
+ .Union(latestStagByVersion.Keys, StringComparer.OrdinalIgnoreCase)
+ .ToList();
+ allVersions.Sort(CompareVersionStrings);
+
+ var avgMsByRunId = runIdsWithData.Count == 0
+ ? new Dictionary()
+ : await ctx.TestResults.AsNoTracking()
+ .Where(r => runIdsWithData.Contains(r.TestRunId))
+ .GroupBy(r => r.TestRunId)
+ .Select(g => new { g.Key, Avg = g.Average(x => (double)x.ResponseTimeMs) })
+ .ToDictionaryAsync(x => x.Key, x => Math.Round(x.Avg, 0), ct);
+
+ static void FillMetrics(
+ TestRunEntity? run,
+ IReadOnlyDictionary avgByRun,
+ out double rate,
+ out double avgMs,
+ out int answered,
+ out int noAnswer,
+ out int errors)
+ {
+ if (run == null)
+ {
+ rate = 0;
+ avgMs = 0;
+ answered = 0;
+ noAnswer = 0;
+ errors = 0;
+ return;
+ }
+
+ var t = run.TotalQuestions;
+ answered = run.SuccessCount;
+ noAnswer = run.NoAnswerCount;
+ errors = run.ErrorCount;
+ rate = t > 0 ? Math.Round(answered * 100.0 / t, 1) : 0;
+ avgMs = avgByRun.TryGetValue(run.Id, out var a) ? a : 0;
+ }
+
+ var versionCompare = new List();
+ foreach (var v in allVersions)
+ {
+ latestProdByVersion.TryGetValue(v, out var pr);
+ latestStagByVersion.TryGetValue(v, out var sr);
+ FillMetrics(pr, avgMsByRunId, out var pRate, out var pAvg, out var pAns, out var pNa, out var pErr);
+ FillMetrics(sr, avgMsByRunId, out var sRate, out var sAvg, out var sAns, out var sNa, out var sErr);
+ versionCompare.Add(new VersionCompareRow(
+ v, pRate, sRate, pAvg, sAvg, pAns, sAns, pNa, sNa, pErr, sErr));
+ }
+
+ var currentV = (primaryRun.IvyVersion ?? "").Trim();
+ latestStagByVersion.TryGetValue(currentV, out var stagingForVersion);
+ latestProdByVersion.TryGetValue(currentV, out var productionForVersion);
+
+ TestRunEntity? peerRun = null;
+ string? peerEnv = null;
+ if (primaryEnv == "production" && stagingForVersion != null)
+ {
+ peerRun = stagingForVersion;
+ peerEnv = "staging";
+ }
+ else if (primaryEnv == "staging" && productionForVersion != null)
+ {
+ peerRun = productionForVersion;
+ peerEnv = "production";
+ }
+
+ var trendDict = primaryEnv == "staging" ? latestStagByVersion : latestProdByVersion;
+ var versionTrendPrimary = trendDict.Values
+ .Select(r =>
+ {
+ var t = r.TotalQuestions;
+ var ans = r.SuccessCount;
+ var rate = t > 0 ? Math.Round(ans * 100.0 / t, 1) : 0.0;
+ var avg = avgMsByRunId.TryGetValue(r.Id, out var a) ? (int)Math.Round(a) : 0;
+ return (Version: (r.IvyVersion ?? "").Trim(), rate, avgMs: avg);
+ })
+ .OrderBy(x => x.Version, Comparer.Create((a, b) => CompareVersionStrings(a, b)))
+ .ToList();
+
+ var latestResults = await ctx.TestResults.AsNoTracking()
+ .AsSplitQuery()
+ .Include(r => r.Question)
+ .Where(r => r.TestRunId == primaryRun.Id)
+ .ToListAsync(ct);
+
+ if (latestResults.Count == 0 && (primaryRun.CompletedAt == null || primaryRun.TotalQuestions == 0))
+ return null;
+
+ double? prevAnswerRate = null;
+ int? prevAvgMs = null;
+ if (peerRun == null)
+ {
+ var idx = versionTrendPrimary.FindIndex(r =>
+ string.Equals(r.Version, currentV, StringComparison.OrdinalIgnoreCase));
+ if (idx > 0)
+ {
+ var prev = versionTrendPrimary[idx - 1];
+ prevAnswerRate = prev.rate;
+ prevAvgMs = prev.avgMs;
+ }
+ else
+ {
+ // Cannot call NormalizeEnvironment inside IQueryable — EF cannot translate it to SQL.
+ var prevRunBase = ctx.TestRuns.AsNoTracking()
+ .Where(r =>
+ r.StartedAt < primaryRun.StartedAt
+ && r.CompletedAt != null
+ && runIdsWithData.Contains(r.Id));
+ var prevRunQuery = string.Equals(primaryEnv, "staging", StringComparison.Ordinal)
+ ? prevRunBase.Where(r => (r.Environment ?? "").Trim().ToLower() == "staging")
+ : prevRunBase.Where(r => (r.Environment ?? "").Trim().ToLower() != "staging");
+ var prevRun = await prevRunQuery
+ .OrderByDescending(r => r.StartedAt)
+ .FirstOrDefaultAsync(ct);
+ if (prevRun != null)
+ {
+ var prevResults = await ctx.TestResults.AsNoTracking()
+ .Where(r => r.TestRunId == prevRun.Id)
+ .ToListAsync(ct);
+ if (prevRun.TotalQuestions > 0 && prevResults.Count == prevRun.TotalQuestions)
+ {
+ var prevAns = prevResults.Count(r => r.IsSuccess);
+ prevAnswerRate = Math.Round(prevAns * 100.0 / prevResults.Count, 1);
+ prevAvgMs = (int)prevResults.Average(r => r.ResponseTimeMs);
+ }
+ else if (prevRun.TotalQuestions > 0)
+ {
+ prevAnswerRate = Math.Round(prevRun.SuccessCount * 100.0 / prevRun.TotalQuestions, 1);
+ prevAvgMs = prevResults.Count > 0 ? (int)prevResults.Average(r => r.ResponseTimeMs) : 0;
+ }
+ }
+ }
+ }
+
+ var detail = BuildDashboardData(latestResults, prevAnswerRate, prevAvgMs, primaryRun);
+
+ DashboardData? peerDetail = null;
+ if (peerRun != null)
+ {
+ var peerResults = await ctx.TestResults.AsNoTracking()
+ .AsSplitQuery()
+ .Include(r => r.Question)
+ .Where(r => r.TestRunId == peerRun.Id)
+ .ToListAsync(ct);
+ peerDetail = BuildDashboardData(peerResults, null, null, peerRun);
+ }
+
+ var allRuns = runs.Select(r =>
+ {
+ var t = r.TotalQuestions;
+ var rate = t > 0 ? Math.Round(r.SuccessCount * 100.0 / t, 1) : 0.0;
+ var avg = avgMsByRunId.TryGetValue(r.Id, out var a) ? (int)Math.Round(a) : 0;
+ return new TestRunRow(
+ r.Id,
+ r.IvyVersion ?? "",
+ CapitalizeEnv(r.Environment ?? "production"),
+ r.DifficultyFilter ?? "all",
+ r.TotalQuestions,
+ r.SuccessCount,
+ r.NoAnswerCount,
+ r.ErrorCount,
+ rate,
+ avg,
+ r.StartedAt.ToLocalTime(),
+ r.CompletedAt?.ToLocalTime());
+ }).ToList();
+
+ return new DashboardPageModel(
+ currentV,
+ primaryRun.StartedAt,
+ primaryEnv,
+ detail,
+ peerDetail,
+ peerEnv,
+ versionCompare,
+ allRuns);
+ }
+
+ private static DashboardData BuildDashboardData(
+ List results,
+ double? prevAnswerRate,
+ int? prevAvgMs,
+ TestRunEntity? run = null)
+ {
+ var useRunSummary = run != null && run.TotalQuestions > 0
+ && (results.Count == 0 || results.Count != run.TotalQuestions);
+
+ int total, answered, noAnswer, errors;
+ double answerRate;
+ int avgMs;
+ if (useRunSummary)
+ {
+ total = run!.TotalQuestions;
+ answered = run.SuccessCount;
+ noAnswer = run.NoAnswerCount;
+ errors = run.ErrorCount;
+ answerRate = total > 0 ? Math.Round(answered * 100.0 / total, 1) : 0;
+ avgMs = results.Count > 0 ? (int)results.Average(r => r.ResponseTimeMs) : 0;
+ }
+ else if (results.Count == 0)
+ {
+ return new DashboardData(
+ 0, 0, 0, 0, 0, 0, prevAnswerRate, prevAvgMs,
+ [], []);
+ }
+ else
+ {
+ total = results.Count;
+ answered = results.Count(r => r.IsSuccess);
+ noAnswer = results.Count(r => !r.IsSuccess && r.HttpStatus == 404);
+ errors = results.Count(r => !r.IsSuccess && r.HttpStatus != 404);
+ answerRate = total > 0 ? Math.Round(answered * 100.0 / total, 1) : 0;
+ avgMs = (int)results.Average(r => r.ResponseTimeMs);
+ }
+
+ var widgetGroups = results
+ .GroupBy(r => r.Question.Widget)
+ .Select(g =>
+ {
+ var t = g.Count();
+ var a = g.Count(r => r.IsSuccess);
+ var rate = t > 0 ? Math.Round(a * 100.0 / t, 1) : 0;
+ var avg = (int)g.Average(r => r.ResponseTimeMs);
+ var max = g.Max(r => r.ResponseTimeMs);
+ return new WidgetProblem(g.Key, rate, t - a, t, avg, max);
+ })
+ .ToList();
+
+ var worstWidgets = widgetGroups.OrderBy(w => w.AnswerRate).Take(10).ToList();
+
+ var diffBreakdown = results
+ .GroupBy(r => r.Question.Difficulty)
+ .Select(g =>
+ {
+ var t = g.Count();
+ var a = g.Count(r => r.IsSuccess);
+ var na = g.Count(r => !r.IsSuccess && r.HttpStatus == 404);
+ var err = g.Count(r => !r.IsSuccess && r.HttpStatus != 404);
+ var rate = t > 0 ? Math.Round(a * 100.0 / t, 1) : 0;
+ return new DifficultyRow(g.Key, rate, a, na, err, t);
+ })
+ .OrderBy(d => d.Difficulty == "easy" ? 0 : d.Difficulty == "medium" ? 1 : 2)
+ .ToList();
+
+ return new DashboardData(
+ total, answered, noAnswer, errors, answerRate, avgMs,
+ prevAnswerRate, prevAvgMs,
+ worstWidgets, diffBreakdown);
+ }
+
+ /// Semantic-ish ordering so 1.2.26 < 1.2.27 < 1.10.0.
+ internal static int CompareVersionStrings(string? a, string? b)
+ {
+ if (string.Equals(a, b, StringComparison.OrdinalIgnoreCase)) return 0;
+ if (string.IsNullOrEmpty(a)) return -1;
+ if (string.IsNullOrEmpty(b)) return 1;
+
+ var pa = a.Split('.', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
+ var pb = b.Split('.', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
+ var n = Math.Max(pa.Length, pb.Length);
+ for (var i = 0; i < n; i++)
+ {
+ var sa = i < pa.Length ? pa[i] : "";
+ var sb = i < pb.Length ? pb[i] : "";
+ var na = int.TryParse(sa, out var ia) ? ia : int.MinValue;
+ var nb = int.TryParse(sb, out var ib) ? ib : int.MinValue;
+ if (na != int.MinValue && nb != int.MinValue)
+ {
+ if (na != nb) return na.CompareTo(nb);
+ continue;
+ }
+
+ var cmp = string.Compare(sa, sb, StringComparison.OrdinalIgnoreCase);
+ if (cmp != 0) return cmp;
+ }
+
+ return string.Compare(a, b, StringComparison.OrdinalIgnoreCase);
+ }
+}
+
+internal sealed class DashboardVersionPickerSheet(
+ IState isOpen,
+ IState dashboardFocusVersion,
+ IReadOnlyList rows,
+ string currentDisplayedVersion) : ViewBase
+{
+ private static readonly Comparer VersionComparerDescending =
+ Comparer.Create((a, b) => DashboardApp.CompareVersionStrings(a, b));
+
+ /// One headline success rate: average when both envs have data, otherwise the single non-zero side.
+ private static string FormatCombinedAnswerRate(VersionCompareRow row)
+ {
+ var p = row.ProductionAnswerRate;
+ var s = row.StagingAnswerRate;
+ if (p <= 0 && s <= 0) return "—";
+ if (s <= 0) return $"{p:F1}%";
+ if (p <= 0) return $"{s:F1}%";
+ return $"{(p + s) / 2.0:F1}%";
+ }
+
+ public override object? Build()
+ {
+ var versionSearch = UseState("");
+
+ if (!isOpen.Value)
+ return null;
+
+ var q = versionSearch.Value.Trim();
+ var filteredRows = (string.IsNullOrEmpty(q)
+ ? rows
+ : rows.Where(r => r.Version.Contains(q, StringComparison.OrdinalIgnoreCase)))
+ .OrderByDescending(r => r.Version, VersionComparerDescending)
+ .ToList();
+
+ var listItems = new List();
+ foreach (var row in filteredRows)
+ {
+ var v = row.Version;
+ var isCurrent = string.Equals(v, currentDisplayedVersion, StringComparison.OrdinalIgnoreCase);
+ var title = isCurrent ? $"{v} (current)" : v;
+ listItems.Add(new ListItem(
+ title: title,
+ subtitle: FormatCombinedAnswerRate(row),
+ onClick: () =>
+ {
+ dashboardFocusVersion.Set(v);
+ isOpen.Set(false);
+ }));
+ }
+
+ var sheetDescription = string.IsNullOrEmpty(q)
+ ? $"{rows.Count} version(s) with test data"
+ : $"{filteredRows.Count} matching · {rows.Count} total";
+
+ // Sheet body is a fixed column: intro + search stay put; only the list scrolls (flex child with Grow).
+ var body = Layout.Vertical().Height(Size.Full())
+ | versionSearch.ToSearchInput().Placeholder("Search versions…").Width(Size.Full())
+ | (!string.IsNullOrEmpty(q) && filteredRows.Count == 0
+ ? (object)Text.Block("No versions match your search — try another query or clear the filter.").Muted()
+ : new Empty())
+ | new List(listItems.ToArray()).Height(Size.Full());
+
+ return new Sheet(
+ _ => isOpen.Set(false),
+ body,
+ title: "Ivy version",
+ description: sheetDescription)
+ .Width(Size.Fraction(0.28f))
+ .Height(Size.Full());
+ }
+}
+
+internal record DashboardPageModel(
+ string IvyVersion,
+ DateTime RunStartedAt,
+ string PrimaryEnvironment,
+ DashboardData Detail,
+ DashboardData? PeerDetail,
+ string? PeerEnvironment,
+ List VersionCompare,
+ List AllRuns);
+
+internal record TestRunRow(
+ Guid Id,
+ string IvyVersion,
+ string Environment,
+ string DifficultyFilter,
+ int TotalQuestions,
+ int SuccessCount,
+ int NoAnswerCount,
+ int ErrorCount,
+ double AnswerRate,
+ int AvgMs,
+ DateTime StartedAt,
+ DateTime? CompletedAt);
+
+/// Per Ivy version: latest production vs latest staging completed runs (0 when an env has no run).
+internal record VersionCompareRow(
+ string Version,
+ double ProductionAnswerRate,
+ double StagingAnswerRate,
+ double ProductionAvgMs,
+ double StagingAvgMs,
+ int ProductionAnswered,
+ int StagingAnswered,
+ int ProductionNoAnswer,
+ int StagingNoAnswer,
+ int ProductionErrors,
+ int StagingErrors);
+
+internal record DashboardData(
+ int Total, int Answered, int NoAnswer, int Errors,
+ double AnswerRate, int AvgMs,
+ double? PrevAnswerRate, int? PrevAvgMs,
+ List WorstWidgets,
+ List DifficultyBreakdown);
+
+internal record WidgetProblem(string Widget, double AnswerRate, int Failed, int Tested, int AvgMs, int MaxMs);
+internal record DifficultyRow(string Difficulty, double Rate, int Answered, int NoAnswer, int Errors, int Total);
diff --git a/project-demos/ivy-ask-statistics/Apps/QuestionEditSheet.cs b/project-demos/ivy-ask-statistics/Apps/QuestionEditSheet.cs
new file mode 100644
index 00000000..f5b1123d
--- /dev/null
+++ b/project-demos/ivy-ask-statistics/Apps/QuestionEditSheet.cs
@@ -0,0 +1,204 @@
+namespace IvyAskStatistics.Apps;
+
+internal sealed class QuestionEditSheet(
+ IState isOpen,
+ Guid questionId,
+ IState previewResultId) : ViewBase
+{
+ private record EditRequest
+ {
+ [Required]
+ public string QuestionText { get; init; } = "";
+
+ [Required]
+ public string Difficulty { get; init; } = "";
+
+ public string Category { get; init; } = "";
+
+ public bool IsActive { get; init; } = true;
+ }
+
+ ///
+ /// Question row plus optional response preview: either the specific row
+ /// (when opened from a run results table) or the latest answer across all runs (elsewhere).
+ ///
+ private sealed record QuestionEditPayload(
+ QuestionEntity Question,
+ string? AnswerText,
+ AnswerPreviewSource PreviewSource);
+
+ private enum AnswerPreviewSource
+ {
+ /// This run's result row (may be empty).
+ ThisResultRow,
+
+ /// Latest successful, else latest with text — any run.
+ GlobalHistory,
+ }
+
+ public override object? Build()
+ {
+ var factory = UseService();
+ var queryService = UseService();
+ var isSaving = UseState(false);
+
+ var questionQuery = UseQuery(
+ key: (questionId, previewResultId.Value),
+ fetcher: async (key, ct) =>
+ {
+ var (id, focusResult) = key;
+ await using var ctx = factory.CreateDbContext();
+ var q = await ctx.Questions.AsNoTracking().FirstOrDefaultAsync(x => x.Id == id, ct);
+ if (q == null)
+ return null;
+
+ if (focusResult is Guid fr && fr != Guid.Empty)
+ {
+ var text = await ctx.TestResults.AsNoTracking()
+ .Where(r => r.Id == fr && r.QuestionId == id)
+ .Select(r => r.ResponseText)
+ .FirstOrDefaultAsync(ct);
+ var trimmed = string.IsNullOrWhiteSpace(text) ? null : text.Trim();
+ return new QuestionEditPayload(q, trimmed, AnswerPreviewSource.ThisResultRow);
+ }
+
+ var lastSuccess = await ctx.TestResults.AsNoTracking()
+ .Where(r => r.QuestionId == id && r.IsSuccess)
+ .OrderByDescending(r => r.CreatedAt)
+ .Select(r => r.ResponseText)
+ .FirstOrDefaultAsync(ct);
+
+ string? answer = null;
+ if (!string.IsNullOrWhiteSpace(lastSuccess))
+ answer = lastSuccess.Trim();
+ else
+ {
+ var lastAny = await ctx.TestResults.AsNoTracking()
+ .Where(r => r.QuestionId == id)
+ .OrderByDescending(r => r.CreatedAt)
+ .Select(r => r.ResponseText)
+ .FirstOrDefaultAsync(ct);
+ if (!string.IsNullOrWhiteSpace(lastAny))
+ answer = lastAny.Trim();
+ }
+
+ return new QuestionEditPayload(q, answer, AnswerPreviewSource.GlobalHistory);
+ });
+
+ if (questionQuery.Loading || questionQuery.Value == null)
+ return new Sheet(
+ _ =>
+ {
+ isOpen.Set(false);
+ previewResultId.Set(null);
+ },
+ Skeleton.Form(),
+ title: "Edit Question",
+ description: "Loading question…")
+ .Width(Size.Fraction(1f / 3f))
+ .Height(Size.Full());
+
+ var payload = questionQuery.Value;
+ var q = payload.Question;
+ var answer = payload.AnswerText;
+ var preview = payload.PreviewSource;
+
+ var form = new EditRequest
+ {
+ QuestionText = q.QuestionText ?? "",
+ Difficulty = q.Difficulty,
+ Category = q.Category,
+ IsActive = q.IsActive,
+ };
+
+ var difficulties = new[] { "easy", "medium", "hard" }.ToOptions();
+
+ var formBuilder = form
+ .ToForm()
+ .Builder(f => f.QuestionText, f => f.ToTextareaInput())
+ .Builder(f => f.Difficulty, f => f.ToSelectInput(difficulties))
+ .Builder(f => f.Category, f => f.ToTextInput())
+ .Builder(f => f.IsActive, f => f.ToSwitchInput())
+ .OnSubmit(OnSubmit);
+
+ var (onSubmit, formView, validationView, loading) = formBuilder.UseForm(Context);
+
+ object answerPreview = preview switch
+ {
+ AnswerPreviewSource.ThisResultRow when string.IsNullOrWhiteSpace(answer)
+ => new Callout(
+ "No response text for this row in this test run (e.g. 404 / no answer).",
+ variant: CalloutVariant.Info),
+ AnswerPreviewSource.ThisResultRow
+ => new Card(
+ Layout.Vertical().Gap(2)
+ | Text.Block("Response for the result row you opened (read-only).").Muted()
+ | Text.Markdown(answer!))
+ .Title("This run")
+ .Icon(Icons.FileText),
+ AnswerPreviewSource.GlobalHistory when string.IsNullOrWhiteSpace(answer)
+ => new Empty(),
+ AnswerPreviewSource.GlobalHistory
+ => new Card(
+ Layout.Vertical().Gap(2)
+ | Text.Block("Latest recorded response across all test runs (read-only).").Muted()
+ | Text.Markdown(answer!))
+ .Title("Answer")
+ .Icon(Icons.FileText),
+ _ => new Empty()
+ };
+
+ var scrollBody = Layout.Vertical().Gap(4)
+ | formView
+ | answerPreview;
+
+ var footer = Layout.Horizontal().Gap(2)
+ | new Button("Save")
+ .Variant(ButtonVariant.Primary)
+ .Loading(loading || isSaving.Value)
+ .Disabled(loading || isSaving.Value)
+ .OnClick(async _ =>
+ {
+ isSaving.Set(true);
+ try
+ {
+ if (await onSubmit())
+ isOpen.Set(false);
+ }
+ finally
+ {
+ isSaving.Set(false);
+ }
+ })
+ | validationView;
+
+ var sheetBody = new FooterLayout(footer, scrollBody);
+
+ return new Sheet(
+ _ =>
+ {
+ isOpen.Set(false);
+ previewResultId.Set(null);
+ },
+ sheetBody,
+ title: "Edit Question")
+ .Width(Size.Fraction(1f / 3f))
+ .Height(Size.Full());
+
+ async Task OnSubmit(EditRequest? request)
+ {
+ if (request == null) return;
+ await using var ctx = factory.CreateDbContext();
+ var entity = await ctx.Questions.FirstOrDefaultAsync(e => e.Id == questionId);
+ if (entity == null) return;
+ entity.QuestionText = request.QuestionText.Trim();
+ entity.Difficulty = request.Difficulty;
+ entity.Category = request.Category.Trim();
+ entity.IsActive = request.IsActive;
+ await ctx.SaveChangesAsync();
+ queryService.RevalidateByTag(("widget-questions", entity.Widget));
+ queryService.RevalidateByTag("widget-summary");
+ queryService.RevalidateByTag(RunApp.TestQuestionsQueryTag);
+ }
+ }
+}
diff --git a/project-demos/ivy-ask-statistics/Apps/QuestionsApp.cs b/project-demos/ivy-ask-statistics/Apps/QuestionsApp.cs
new file mode 100644
index 00000000..72404554
--- /dev/null
+++ b/project-demos/ivy-ask-statistics/Apps/QuestionsApp.cs
@@ -0,0 +1,579 @@
+using System.Collections.Immutable;
+using System.Threading;
+
+namespace IvyAskStatistics.Apps;
+
+internal sealed record WidgetTableData(List Rows, List Catalog, string QueryKey);
+
+internal sealed record GenProgress(
+ string CurrentWidget,
+ int Done,
+ int Total,
+ List Failed,
+ bool Active,
+ int MaxParallel = 1);
+
+[App(icon: Icons.Database, title: "Questions")]
+public class QuestionsApp : ViewBase
+{
+ private const string TableQueryKey = "questions-widget-table";
+
+ /// Batch “Generate all” runs this many widgets concurrently (no config).
+ private const int WidgetGenerationParallelism = 4;
+
+ public override object? Build()
+ {
+ var factory = UseService();
+ var configuration = UseService();
+ var client = UseService();
+ var queryService = UseService();
+
+ var generatingWidgets = UseState(ImmutableHashSet.Empty);
+ var deleteRequest = UseState(null);
+ var viewDialogOpen = UseState(false);
+ var viewDialogWidget = UseState("");
+ var editSheetOpen = UseState(false);
+ var editQuestionId = UseState(Guid.Empty);
+ var editPreviewResultId = UseState(null);
+ var refreshToken = UseRefreshToken();
+ var genProgress = UseState(null);
+ var (alertView, showAlert) = UseAlert();
+
+ var tableQuery = UseQuery(
+ key: TableQueryKey,
+ fetcher: async (qk, ct) =>
+ {
+ var result = await LoadWidgetTableDataAsync(factory, qk, ct);
+ refreshToken.Refresh();
+ return result;
+ },
+ options: new QueryOptions { KeepPrevious = true, RefreshInterval = TimeSpan.FromSeconds(10), RevalidateOnMount = true },
+ tags: ["widget-summary"]);
+
+ // Register flush on every build (fresh showAlert closure). Call Flush after bind so a pending
+ // request that arrived before this view existed still opens the dialog. Request() also invokes
+ // the handler so repeat footer clicks work when Navigate does not rebuild (same tab).
+ UseEffect(() =>
+ {
+ void FlushFooterGenerateAll()
+ {
+ if (!GenerateAllBridge.Consume()) return;
+ ShowFooterGenerateAllDialog();
+ }
+
+ GenerateAllBridge.SetFlushHandler(FlushFooterGenerateAll);
+ FlushFooterGenerateAll();
+ }, EffectTrigger.OnBuild());
+
+ UseEffect(() => new GenerateAllFlushRegistration(), EffectTrigger.OnMount());
+
+ UseEffect(async () =>
+ {
+ var widgetName = deleteRequest.Value;
+ if (string.IsNullOrEmpty(widgetName)) return;
+
+ try
+ {
+ await using var ctx = factory.CreateDbContext();
+ var list = await ctx.Questions.Where(q => q.Widget == widgetName).ToListAsync();
+ if (list.Count == 0) return;
+ ctx.Questions.RemoveRange(list);
+ await ctx.SaveChangesAsync();
+
+ var fresh = await LoadWidgetTableDataAsync(factory, TableQueryKey, CancellationToken.None);
+ tableQuery.Mutator.Mutate(fresh, revalidate: false);
+ refreshToken.Refresh();
+ queryService.RevalidateByTag(RunApp.TestQuestionsQueryTag);
+ }
+ catch
+ {
+ // best-effort
+ }
+ finally
+ {
+ deleteRequest.Set(null);
+ }
+ }, [deleteRequest.ToTrigger()]);
+
+ async Task GenerateOneAsync(IvyWidget widget)
+ {
+ var apiKey = configuration[QuestionGeneratorService.ApiKeyConfigKey]!.Trim();
+ var baseUrl = configuration[QuestionGeneratorService.BaseUrlConfigKey]!.Trim();
+ await QuestionGeneratorService.GenerateAndSaveAsync(widget, factory, apiKey, baseUrl, configuration);
+ }
+
+ async Task GenerateWidgetAsync(IvyWidget widget)
+ {
+ try
+ {
+ genProgress.Set(new GenProgress(widget.Name, 0, 1, [], true, 1));
+ await GenerateOneAsync(widget);
+
+ var fresh = await LoadWidgetTableDataAsync(factory, TableQueryKey, CancellationToken.None);
+ tableQuery.Mutator.Mutate(fresh, revalidate: false);
+ queryService.RevalidateByTag(RunApp.TestQuestionsQueryTag);
+ genProgress.Set(new GenProgress(widget.Name, 1, 1, [], false, 1));
+ }
+ catch (Exception ex)
+ {
+ client.Toast(ex.Message);
+ genProgress.Set(new GenProgress(widget.Name, 0, 1, [widget.Name], false, 1));
+ }
+ finally
+ {
+ generatingWidgets.Set(s => s.Remove(widget.Name));
+ refreshToken.Refresh();
+ }
+ }
+
+ async Task GenerateBatchAsync(List widgets)
+ {
+ const int maxRetries = 2;
+ var maxParallel = WidgetGenerationParallelism;
+ var completed = 0;
+ var failedLock = new object();
+ var failed = new List();
+
+ using var sem = new SemaphoreSlim(maxParallel);
+ using var uiGate = new SemaphoreSlim(1, 1);
+ using var tickerCts = new CancellationTokenSource();
+ using var ticker = new PeriodicTimer(TimeSpan.FromMilliseconds(800));
+
+ async Task PushUiFromStateAsync()
+ {
+ await uiGate.WaitAsync();
+ try
+ {
+ var d = Volatile.Read(ref completed);
+ List failedCopy;
+ lock (failedLock)
+ failedCopy = [..failed];
+ genProgress.Set(new GenProgress("", d, widgets.Count, failedCopy, true, maxParallel));
+ var fresh = await LoadWidgetTableDataAsync(factory, TableQueryKey, CancellationToken.None);
+ tableQuery.Mutator.Mutate(fresh, revalidate: false);
+ refreshToken.Refresh();
+ }
+ finally
+ {
+ uiGate.Release();
+ }
+ }
+
+ var uiTickerTask = Task.Run(async () =>
+ {
+ try
+ {
+ while (await ticker.WaitForNextTickAsync(tickerCts.Token))
+ await PushUiFromStateAsync();
+ }
+ catch (OperationCanceledException)
+ {
+ // expected when batch finishes
+ }
+ });
+
+ try
+ {
+ await PushUiFromStateAsync();
+
+ var workerTasks = widgets.Select(async widget =>
+ {
+ await sem.WaitAsync();
+ try
+ {
+ var success = false;
+ Exception? lastEx = null;
+ for (var attempt = 1; attempt <= maxRetries && !success; attempt++)
+ {
+ try
+ {
+ await GenerateOneAsync(widget);
+ success = true;
+ }
+ catch (Exception ex)
+ {
+ lastEx = ex;
+ if (attempt < maxRetries)
+ await Task.Delay(2000);
+ }
+ }
+
+ if (success)
+ {
+ Interlocked.Increment(ref completed);
+ queryService.RevalidateByTag(RunApp.TestQuestionsQueryTag);
+ }
+ else
+ {
+ lock (failedLock)
+ failed.Add(widget.Name);
+ if (lastEx != null)
+ client.Toast($"\"{widget.Name}\": {lastEx.Message}");
+ }
+
+ await uiGate.WaitAsync();
+ try
+ {
+ generatingWidgets.Set(s => s.Remove(widget.Name));
+ refreshToken.Refresh();
+ }
+ finally
+ {
+ uiGate.Release();
+ }
+ }
+ finally
+ {
+ sem.Release();
+ }
+ }).ToArray();
+
+ await Task.WhenAll(workerTasks);
+ }
+ finally
+ {
+ tickerCts.Cancel();
+ try
+ {
+ await uiTickerTask;
+ }
+ catch
+ {
+ // ignore cancellation teardown
+ }
+
+ ticker.Dispose();
+ }
+
+ await uiGate.WaitAsync();
+ try
+ {
+ generatingWidgets.Set(_ => ImmutableHashSet.Empty);
+ List failedCopy;
+ lock (failedLock)
+ failedCopy = [..failed];
+ genProgress.Set(new GenProgress(
+ "",
+ Volatile.Read(ref completed),
+ widgets.Count,
+ failedCopy,
+ false,
+ maxParallel));
+ var finalFresh = await LoadWidgetTableDataAsync(factory, TableQueryKey, CancellationToken.None);
+ tableQuery.Mutator.Mutate(finalFresh, revalidate: false);
+ refreshToken.Refresh();
+ queryService.RevalidateByTag(RunApp.TestQuestionsQueryTag);
+ }
+ finally
+ {
+ uiGate.Release();
+ }
+ }
+
+ void MarkGenerating(IEnumerable widgetNames)
+ {
+ var names = widgetNames.Where(n => !string.IsNullOrEmpty(n)).ToHashSet();
+ if (names.Count == 0) return;
+ generatingWidgets.Set(s => s.Union(names).ToImmutableHashSet());
+ refreshToken.Refresh();
+ }
+
+ void ShowFooterGenerateAllDialog()
+ {
+ showAlert(
+ "Generate questions for all widgets that don't have questions yet?\n\nOpenAI will be called 3 times per widget (easy / medium / hard). The widget list loads after you tap OK; only widgets with no questions are generated.",
+ result =>
+ {
+ if (!result.IsOk()) return;
+ var cfgErr = QuestionGeneratorService.GetOpenAiConfigurationError(configuration);
+ if (cfgErr != null)
+ {
+ client.Toast(cfgErr);
+ return;
+ }
+
+ _ = RunGenerateAllAfterConfirmAsync();
+ },
+ "Generate All Questions",
+ AlertButtonSet.OkCancel);
+ }
+
+ async Task RunGenerateAllAfterConfirmAsync()
+ {
+ try
+ {
+ var data = await LoadWidgetTableDataAsync(factory, TableQueryKey, CancellationToken.None);
+ tableQuery.Mutator.Mutate(data, revalidate: false);
+ refreshToken.Refresh();
+
+ var allWidgets = data.Catalog;
+ if (allWidgets.Count == 0)
+ {
+ client.Toast("No widgets found. Check MCP docs or your database.");
+ return;
+ }
+
+ var notGenerated = allWidgets
+ .Where(w => !data.Rows.Any(r => r.Widget == w.Name && r.Easy + r.Medium + r.Hard > 0))
+ .ToList();
+
+ if (notGenerated.Count == 0)
+ {
+ client.Toast("Every widget already has at least one question.");
+ return;
+ }
+
+ MarkGenerating(notGenerated.Select(w => w.Name));
+ genProgress.Set(new GenProgress("", 0, notGenerated.Count, [], true, WidgetGenerationParallelism));
+ _ = GenerateBatchAsync(notGenerated);
+ }
+ catch (Exception ex)
+ {
+ client.Toast(ex.Message);
+ }
+ }
+
+ var generating = generatingWidgets.Value;
+ var baseRows = tableQuery.Value?.Rows ?? [];
+ var catalog = tableQuery.Value?.Catalog ?? [];
+ var isDeleting = !string.IsNullOrEmpty(deleteRequest.Value);
+ var firstLoad = tableQuery.Loading && tableQuery.Value == null;
+ var progress = genProgress.Value;
+ var isGenerating = generating.Count > 0;
+
+ static string IdleStatus(WidgetRow r)
+ {
+ var n = r.Easy + r.Medium + r.Hard;
+ return n == 0 ? "○ Not generated" : "✓ Generated";
+ }
+
+ var rows = baseRows.Select(r =>
+ generating.Contains(r.Widget)
+ ? r with { Status = "Generating…" }
+ : r with { Status = IdleStatus(r) }
+ ).ToList();
+
+ if (firstLoad)
+ return Layout.Vertical().Height(Size.Full())
+ | alertView
+ | TabLoadingSkeletons.QuestionsTab();
+
+ var notGeneratedCount = baseRows.Count(r => r.Easy + r.Medium + r.Hard == 0);
+
+ object progressBar;
+ if (progress is { Active: true })
+ {
+ var pct = progress.Total > 0 ? progress.Done * 100 / progress.Total : 0;
+ var statusLine = progress.MaxParallel > 1
+ ? $"Completed {progress.Done}/{progress.Total} · up to {progress.MaxParallel} widgets in parallel"
+ : progress.Total == 1
+ ? $"Generating questions for {progress.CurrentWidget}…"
+ : $"Generating {Math.Min(progress.Done + 1, progress.Total)}/{progress.Total}: {progress.CurrentWidget}…";
+ progressBar = new Callout(
+ Layout.Vertical()
+ | Text.Block(statusLine)
+ | new Progress(pct).Goal($"{progress.Done}/{progress.Total}"),
+ variant: CalloutVariant.Info);
+ }
+ else if (progress is { Active: false, Total: > 0 })
+ {
+ var failCount = progress.Failed.Count;
+ progressBar = new Callout(
+ Layout.Horizontal()
+ | Text.Block(failCount == 0
+ ? $"Done! Generated questions for {progress.Done}/{progress.Total} widget(s)."
+ : $"Completed: {progress.Done}/{progress.Total} succeeded. Failed: {string.Join(", ", progress.Failed)}")
+ | new Button("Dismiss", onClick: _ => genProgress.Set(null)).Small(),
+ variant: failCount == 0 ? CalloutVariant.Success : CalloutVariant.Warning);
+ }
+ else
+ {
+ progressBar = Text.Muted("");
+ }
+
+ var table = rows.AsQueryable()
+ .ToDataTable(r => r.Widget)
+ .RefreshToken(refreshToken)
+ .Key("questions-widgets")
+ .Height(Size.Full())
+ .Header(r => r.Widget, "Widget")
+ .Header(r => r.Category, "Category")
+ .Header(r => r.Easy, "Easy")
+ .Header(r => r.Medium, "Medium")
+ .Header(r => r.Hard, "Hard")
+ .Header(r => r.LastUpdated, "Last Generated")
+ .Header(r => r.Status, "Status")
+ .Width(r => r.Widget, Size.Px(160))
+ .Width(r => r.Category, Size.Px(120))
+ .Width(r => r.Easy, Size.Px(60))
+ .Width(r => r.Medium, Size.Px(70))
+ .Width(r => r.Hard, Size.Px(60))
+ .Width(r => r.LastUpdated, Size.Px(170))
+ .Width(r => r.Status, Size.Px(280))
+ .RowActions(
+ MenuItem.Default(Icons.List, "questions").Label("View questions").Tag("questions"),
+ MenuItem.Default(Icons.Sparkles, "generate").Label("Generate questions").Tag("generate"),
+ MenuItem.Default(Icons.Trash2, "delete").Label("Delete questions").Tag("delete"))
+ .OnRowAction(e =>
+ {
+ var args = e.Value;
+ var tag = args?.Tag?.ToString();
+ if (string.IsNullOrEmpty(tag)) return ValueTask.CompletedTask;
+
+ if (tag == "questions")
+ {
+ var viewName = args.Id?.ToString() ?? "";
+ if (string.IsNullOrEmpty(viewName)) return ValueTask.CompletedTask;
+ viewDialogWidget.Set(viewName);
+ viewDialogOpen.Set(true);
+ return ValueTask.CompletedTask;
+ }
+
+ if (tag == "generate")
+ {
+ if (isDeleting || isGenerating) return ValueTask.CompletedTask;
+
+ var genName = args.Id?.ToString() ?? "";
+ if (generating.Contains(genName)) return ValueTask.CompletedTask;
+
+ var widget = catalog.FirstOrDefault(w => w.Name == genName)
+ ?? new IvyWidget(genName, rows.FirstOrDefault(r => r.Widget == genName)?.Category ?? "", "");
+
+ showAlert(
+ $"Generate 30 questions for the \"{widget.Name}\" widget?\n\nOpenAI will be called three times (easy / medium / hard). Any previously generated questions for this widget will be replaced.",
+ result =>
+ {
+ if (!result.IsOk()) return;
+ var cfgErr = QuestionGeneratorService.GetOpenAiConfigurationError(configuration);
+ if (cfgErr != null)
+ {
+ client.Toast(cfgErr);
+ return;
+ }
+
+ MarkGenerating([widget.Name]);
+ _ = GenerateWidgetAsync(widget);
+ },
+ "Generate questions",
+ AlertButtonSet.OkCancel);
+ return ValueTask.CompletedTask;
+ }
+
+ if (tag == "delete")
+ {
+ if (isDeleting || isGenerating) return ValueTask.CompletedTask;
+
+ var delName = args.Id?.ToString() ?? "";
+ if (generating.Contains(delName)) return ValueTask.CompletedTask;
+
+ var row = rows.FirstOrDefault(r => r.Widget == delName);
+ var n = row == null ? 0 : row.Easy + row.Medium + row.Hard;
+ if (n == 0) return ValueTask.CompletedTask;
+
+ showAlert(
+ $"Delete all {n} question(s) for the \"{delName}\" widget?\n\nThis cannot be undone.",
+ result =>
+ {
+ if (!result.IsOk()) return;
+ deleteRequest.Set(delName);
+ },
+ "Delete questions",
+ AlertButtonSet.OkCancel);
+ return ValueTask.CompletedTask;
+ }
+
+ return ValueTask.CompletedTask;
+ })
+ .Config(config =>
+ {
+ config.AllowSorting = true;
+ config.AllowFiltering = true;
+ config.ShowSearch = true;
+ config.ShowIndexColumn = false;
+ });
+
+ object? questionsDialog = viewDialogOpen.Value && !string.IsNullOrEmpty(viewDialogWidget.Value)
+ ? new WidgetQuestionsDialog(
+ viewDialogOpen,
+ viewDialogWidget.Value,
+ editSheetOpen,
+ editQuestionId,
+ editPreviewResultId)
+ : null;
+
+ object? editSheet = editSheetOpen.Value && editQuestionId.Value != Guid.Empty
+ ? new QuestionEditSheet(editSheetOpen, editQuestionId.Value, editPreviewResultId)
+ : null;
+
+ return Layout.Vertical().Height(Size.Full())
+ | alertView
+ | progressBar
+ | table
+ | questionsDialog
+ | editSheet;
+ }
+
+ private static async Task LoadWidgetTableDataAsync(
+ AppDbContextFactory factory,
+ string queryKey,
+ CancellationToken ct)
+ {
+ await using var ctx = factory.CreateDbContext();
+
+ var grouped = await ctx.Questions
+ .AsNoTracking()
+ .GroupBy(q => new { q.Widget, q.Category, q.Difficulty })
+ .Select(g => new
+ {
+ g.Key.Widget,
+ g.Key.Category,
+ g.Key.Difficulty,
+ Count = g.Count(),
+ MaxDate = g.Max(x => x.CreatedAt)
+ })
+ .ToListAsync(ct);
+
+ var countsByWidget = grouped
+ .GroupBy(x => x.Widget)
+ .ToDictionary(
+ g => g.Key,
+ g => (
+ category: g.Select(x => x.Category).FirstOrDefault() ?? "",
+ easy: g.FirstOrDefault(x => x.Difficulty == "easy")?.Count ?? 0,
+ medium: g.FirstOrDefault(x => x.Difficulty == "medium")?.Count ?? 0,
+ hard: g.FirstOrDefault(x => x.Difficulty == "hard")?.Count ?? 0,
+ updatedAt: g.Max(x => x.MaxDate)
+ ));
+
+ List catalog = [];
+ try { catalog = await IvyAskService.GetWidgetsAsync(); }
+ catch { }
+
+ var byName = catalog.ToDictionary(w => w.Name);
+ foreach (var (widget, info) in countsByWidget)
+ if (!byName.ContainsKey(widget))
+ byName[widget] = new IvyWidget(widget, info.category, "");
+
+ var rows = byName.Values
+ .OrderBy(w => string.IsNullOrEmpty(w.Category) ? "zzz" : w.Category)
+ .ThenBy(w => w.Name)
+ .Select(w =>
+ {
+ var c = countsByWidget.GetValueOrDefault(w.Name);
+ var category = string.IsNullOrEmpty(w.Category) ? "Unclassified" : w.Category;
+ var updated = c.updatedAt == default
+ ? "—"
+ : c.updatedAt.ToLocalTime().ToString("dd MMM yyyy, HH:mm");
+ return new WidgetRow(w.Name, category, c.easy, c.medium, c.hard, updated, "");
+ })
+ .ToList();
+
+ return new WidgetTableData(rows, catalog, queryKey);
+ }
+
+ /// Clears when the Questions view unmounts.
+ sealed class GenerateAllFlushRegistration : IDisposable
+ {
+ public void Dispose() => GenerateAllBridge.SetFlushHandler(null);
+ }
+}
diff --git a/project-demos/ivy-ask-statistics/Apps/RunApp.cs b/project-demos/ivy-ask-statistics/Apps/RunApp.cs
new file mode 100644
index 00000000..7c87ce09
--- /dev/null
+++ b/project-demos/ivy-ask-statistics/Apps/RunApp.cs
@@ -0,0 +1,808 @@
+using System.Collections.Concurrent;
+using System.Collections.Immutable;
+
+namespace IvyAskStatistics.Apps;
+
+[App(icon: Icons.ChartBar, title: "Run Tests")]
+public class RunApp : ViewBase
+{
+ /// Invalidate with when rows in Questions change.
+ internal const string TestQuestionsQueryTag = "test-questions-db";
+
+ internal const string LastSavedRunQueryTag = "last-saved-run";
+
+ ///
+ /// Must differ from other apps' keys (e.g. Dashboard uses its own string).
+ /// Reusing 0 across tabs overwrote the server query cache and cleared this panel after navigation.
+ ///
+ private const string LastSavedRunQueryKey = "run-tests-last-saved-db";
+
+ private static LastSavedRunSummary? s_lastSuccessfulLastSavedRun;
+
+ /// Fixed parallel Ask requests for every run (not user-configurable).
+ private const int RunParallelism = 20;
+
+ private static readonly string[] DifficultyOptions = ["all", "easy", "medium", "hard"];
+ private static readonly string[] McpEnvironmentOptions = ["production", "staging"];
+
+ private static string McpBaseUrl(string environment) =>
+ environment.Equals("staging", StringComparison.OrdinalIgnoreCase)
+ ? "https://staging.mcp.ivy.app"
+ : "https://mcp.ivy.app";
+
+ public override object? Build()
+ {
+ var factory = UseService();
+ var configuration = UseService();
+ var client = UseService();
+ var queryService = UseService();
+
+ var ivyVersion = UseState(RunTestFormPreferences.IvyVersion);
+ var mcpEnvironment = UseState(RunTestFormPreferences.McpEnvironment);
+ var difficultyFilter = UseState(RunTestFormPreferences.DifficultyFilter);
+ var isRunning = UseState(false);
+ var completed = UseState>(ImmutableList.Empty);
+ var activeIds = UseState(ImmutableHashSet.Empty);
+ var allQuestions = UseState>([]);
+ var persistToDb = UseState(false);
+ var refreshToken = UseRefreshToken();
+ var runFinished = UseState(false);
+ var runVersionExistsDialogOpen = UseState(false);
+
+ UseEffect(() =>
+ {
+ completed.Set(ImmutableList.Empty);
+ activeIds.Set(ImmutableHashSet.Empty);
+ allQuestions.Set([]);
+ runFinished.Set(false);
+ }, [difficultyFilter.ToTrigger()]);
+
+ UseEffect(() =>
+ {
+ ivyVersion.Set(RunTestFormPreferences.IvyVersion);
+ mcpEnvironment.Set(RunTestFormPreferences.McpEnvironment);
+ difficultyFilter.Set(RunTestFormPreferences.DifficultyFilter);
+ }, EffectTrigger.OnMount());
+
+ UseEffect(() =>
+ {
+ RunTestFormPreferences.Set(ivyVersion.Value, mcpEnvironment.Value, difficultyFilter.Value);
+ }, [ivyVersion.ToTrigger(), mcpEnvironment.ToTrigger(), difficultyFilter.ToTrigger()]);
+
+ UseEffect(() =>
+ {
+ SyncIvyVersionForMcpEnvironment(ivyVersion, mcpEnvironment.Value);
+ }, EffectTrigger.OnMount());
+
+ UseEffect(() =>
+ {
+ SyncIvyVersionForMcpEnvironment(ivyVersion, mcpEnvironment.Value);
+ }, [mcpEnvironment.ToTrigger()]);
+
+ var questionsQuery = UseQuery, string>(
+ key: $"questions-{difficultyFilter.Value}",
+ fetcher: async (_, ct) =>
+ {
+ var result = await LoadQuestionsAsync(factory, difficultyFilter.Value);
+ refreshToken.Refresh();
+ return result;
+ },
+ tags: [TestQuestionsQueryTag],
+ options: new QueryOptions { RevalidateOnMount = true, KeepPrevious = true });
+
+ var lastSavedRunQuery = UseQuery(
+ key: LastSavedRunQueryKey,
+ fetcher: async (_, ct) =>
+ {
+ try
+ {
+ var r = await LoadLastSavedRunAsync(factory, ct);
+ s_lastSuccessfulLastSavedRun = r;
+ refreshToken.Refresh();
+ return r;
+ }
+ catch (OperationCanceledException)
+ {
+ return s_lastSuccessfulLastSavedRun;
+ }
+ },
+ tags: [LastSavedRunQueryTag],
+ options: new QueryOptions { KeepPrevious = true, RevalidateOnMount = true });
+
+ // Latest ivy_ask_test_runs.IvyVersion when the input is still empty (prefs / first visit).
+ UseEffect(() =>
+ {
+ if (lastSavedRunQuery.Loading) return;
+ var summary = lastSavedRunQuery.Value ?? s_lastSuccessfulLastSavedRun;
+ if (summary == null || string.IsNullOrWhiteSpace(summary.IvyVersion)) return;
+ if (!string.IsNullOrWhiteSpace(ivyVersion.Value.Trim())) return;
+ ivyVersion.Set(EffectiveIvyVersionForMcp(summary.IvyVersion.Trim(), mcpEnvironment.Value));
+ }, EffectTrigger.OnBuild());
+
+ // Do not call Mutator.Revalidate() from EffectTrigger.OnMount here: both queries already use
+ // RevalidateOnMount, and an extra OnMount revalidate starts a second fetch that cancels the first,
+ // producing OperationCanceledException + Ivy.QueryService "Fetch failed" warnings.
+
+ var running = isRunning.Value;
+ var firstLoad = questionsQuery.Loading && questionsQuery.Value == null && !running;
+ var questions = running && allQuestions.Value.Count > 0 ? allQuestions.Value : questionsQuery.Value ?? [];
+ var completedList = completed.Value;
+ var active = activeIds.Value;
+
+ var done = completedList.Count;
+ var success = completedList.Count(r => r.Status == "success");
+ var noAnswer = completedList.Count(r => r.Status == "no_answer");
+ var errors = completedList.Count(r => r.Status == "error");
+ var avgMs = done > 0 ? (int)completedList.Average(r => r.ResponseTimeMs) : 0;
+ var progressPct = questions.Count > 0 ? done * 100 / questions.Count : 0;
+
+ var lastSavedEffective = lastSavedRunQuery.Value ?? s_lastSuccessfulLastSavedRun;
+
+ List rows;
+ if (running || done > 0)
+ {
+ var completedById = completedList.ToDictionary(r => r.Question.Id);
+ rows = questions.Select(q =>
+ {
+ if (completedById.TryGetValue(q.Id, out var r))
+ {
+ var icon = r.Status == "success" ? Icons.CircleCheck : Icons.CircleX;
+ return new QuestionRow(q.Id, q.Widget, q.Difficulty, q.Question, icon, ToStatusLabel(r.Status), $"{r.ResponseTimeMs}ms");
+ }
+
+ if (active.Contains(q.Id))
+ return new QuestionRow(q.Id, q.Widget, q.Difficulty, q.Question, Icons.Loader, "running", "");
+
+ return new QuestionRow(q.Id, q.Widget, q.Difficulty, q.Question, Icons.Clock, "pending", "");
+ }).ToList();
+ }
+ else if (lastSavedEffective?.Rows.Count > 0)
+ rows = BuildQuestionRowsFromLastSaved(lastSavedEffective);
+ else
+ rows = questions.Select(q =>
+ new QuestionRow(q.Id, q.Widget, q.Difficulty, q.Question, Icons.Clock, "pending", ""))
+ .ToList();
+
+ async Task OnRunAllClickedAsync()
+ {
+ var snapshot = questionsQuery.Value ?? [];
+ if (snapshot.Count == 0) return;
+
+ var raw = ivyVersion.Value.Trim();
+ if (string.IsNullOrEmpty(raw))
+ {
+ client.Toast("Please enter an Ivy version before running.");
+ return;
+ }
+
+ var version = EffectiveIvyVersionForMcp(raw, mcpEnvironment.Value);
+ if (!string.Equals(ivyVersion.Value.Trim(), version, StringComparison.Ordinal))
+ ivyVersion.Set(version);
+
+ if (await RunExistsAsync(factory, version))
+ {
+ runVersionExistsDialogOpen.Set(true);
+ return;
+ }
+
+ await BeginRunAsync(persistToDatabase: true, replaceExistingRunForVersion: false);
+ }
+
+ async Task BeginRunAsync(bool persistToDatabase, bool replaceExistingRunForVersion)
+ {
+ var snapshot = questionsQuery.Value ?? [];
+ if (snapshot.Count == 0) return;
+
+ var raw = ivyVersion.Value.Trim();
+ if (string.IsNullOrEmpty(raw))
+ {
+ client.Toast("Please enter an Ivy version before running.");
+ return;
+ }
+
+ var version = EffectiveIvyVersionForMcp(raw, mcpEnvironment.Value);
+ if (!string.Equals(ivyVersion.Value.Trim(), version, StringComparison.Ordinal))
+ ivyVersion.Set(version);
+
+ persistToDb.Set(persistToDatabase);
+
+ completed.Set(ImmutableList.Empty);
+ activeIds.Set(ImmutableHashSet.Empty);
+ allQuestions.Set(snapshot);
+ runFinished.Set(false);
+ isRunning.Set(true);
+ refreshToken.Refresh();
+
+ var maxParallel = RunParallelism;
+ var baseUrl = McpBaseUrl(mcpEnvironment.Value);
+ var mcpClient = IvyAskService.ResolveMcpClientId(configuration);
+
+ _ = Task.Run(async () =>
+ {
+ var runStartedUtc = DateTime.UtcNow;
+ var bag = new ConcurrentBag();
+ var inFlight = new ConcurrentDictionary();
+ using var sem = new SemaphoreSlim(maxParallel);
+
+ using var ticker = new PeriodicTimer(TimeSpan.FromMilliseconds(500));
+ var tickerCts = new CancellationTokenSource();
+ var uiTask = Task.Run(async () =>
+ {
+ while (!tickerCts.IsCancellationRequested)
+ {
+ try { await ticker.WaitForNextTickAsync(tickerCts.Token); } catch { break; }
+ completed.Set(_ => bag.ToImmutableList());
+ activeIds.Set(_ => inFlight.Keys.ToImmutableHashSet());
+ refreshToken.Refresh();
+ }
+ });
+
+ var tasks = snapshot.Select(async q =>
+ {
+ await sem.WaitAsync();
+ inFlight[q.Id] = true;
+
+ try
+ {
+ var result = await IvyAskService.AskAsync(q, baseUrl, mcpClient);
+ bag.Add(result);
+ }
+ finally
+ {
+ inFlight.TryRemove(q.Id, out _);
+ sem.Release();
+ }
+ });
+
+ await Task.WhenAll(tasks);
+ tickerCts.Cancel();
+ await uiTask;
+
+ completed.Set(_ => bag.ToImmutableList());
+ activeIds.Set(ImmutableHashSet.Empty);
+
+ var finalResults = OrderResultsLikeSnapshot(snapshot, bag.ToList());
+ if (persistToDatabase)
+ {
+ var saved = await PersistNewRunAsync(
+ factory,
+ version,
+ snapshot,
+ finalResults,
+ runStartedUtc,
+ mcpEnvironment.Value,
+ difficultyFilter.Value,
+ RunParallelism.ToString(),
+ replaceExistingRunForVersion);
+ if (saved)
+ {
+ queryService.RevalidateByTag("dashboard-stats");
+ queryService.RevalidateByTag(LastSavedRunQueryTag);
+ }
+ else
+ client.Toast("Could not save results to the database.");
+ }
+
+ isRunning.Set(false);
+ runFinished.Set(true);
+ refreshToken.Refresh();
+ });
+ }
+
+ object lastSavedRunPanel = BuildLastSavedRunPanel(lastSavedRunQuery, lastSavedEffective);
+
+ var mcpBaseForUi = McpBaseUrl(mcpEnvironment.Value);
+
+ var controls = Layout.Horizontal().Gap(2).Height(Size.Fit())
+ | ivyVersion.ToTextInput()
+ .Placeholder("Ivy version (e.g. 1.2.27; staging adds -staging)")
+ .Disabled(running)
+ | mcpEnvironment.ToSelectInput(McpEnvironmentOptions).Disabled(running)
+ | difficultyFilter.ToSelectInput(DifficultyOptions).Disabled(running)
+ | new Button("Run All", onClick: async _ => await OnRunAllClickedAsync())
+ .Primary()
+ .Icon(Icons.Play)
+ .Disabled(running || questionsQuery.Loading || questions.Count == 0);
+
+ if (firstLoad)
+ return Layout.Vertical().Height(Size.Full())
+ | controls
+ | lastSavedRunPanel
+ | TabLoadingSkeletons.RunTab();
+
+ object statusBar;
+ if (running)
+ {
+ var inFlight = active.Count;
+ statusBar = new Callout(
+ Layout.Vertical()
+ | Text.Block(
+ $"Running {done}/{questions.Count} completed, {inFlight} in flight (x{RunParallelism} parallel) · {mcpBaseForUi}")
+ | new Progress(progressPct).Goal($"{done}/{questions.Count}"),
+ variant: CalloutVariant.Info);
+ }
+ else if (runFinished.Value && done > 0)
+ {
+ var suffix = persistToDb.Value ? "Results saved to database." : "Local only — this version was already tested.";
+ var hasErrors = errors > 0;
+ statusBar = new Callout(
+ Text.Block(hasErrors
+ ? $"Completed: {success}/{done} answered, {noAnswer} no answer, {errors} error(s). {suffix}"
+ : $"Done! {success}/{done} answered, {noAnswer} no answer. {suffix}"),
+ variant: hasErrors ? CalloutVariant.Warning : CalloutVariant.Success);
+ }
+ else
+ {
+ statusBar = Text.Muted("");
+ }
+
+ object kpiCards = done > 0
+ ? BuildOutcomeMetricCards(
+ done,
+ success,
+ noAnswer,
+ errors,
+ avgMs,
+ completedList.Min(r => r.ResponseTimeMs),
+ completedList.Max(r => r.ResponseTimeMs))
+ : Text.Muted("");
+
+ object liveRunKpiWhileRunning = done > 0
+ ? BuildOutcomeMetricCards(
+ done,
+ success,
+ noAnswer,
+ errors,
+ avgMs,
+ completedList.Min(r => r.ResponseTimeMs),
+ completedList.Max(r => r.ResponseTimeMs))
+ : BuildLiveRunMetricPlaceholders(questions.Count);
+
+ // After a finished session, show only this run’s metrics + callout — not the older DB snapshot row.
+ object historyOrActiveRunPanel = running
+ ? Layout.Vertical().Gap(2)
+ | statusBar
+ | liveRunKpiWhileRunning
+ : runFinished.Value && done > 0
+ ? new Empty()
+ : lastSavedRunPanel;
+
+ // Completed run: green status + current KPI cards (no "This run" heading; KPIs are not duplicated above).
+ object afterHistoryPanel = !running && runFinished.Value && done > 0
+ ? Layout.Vertical().Gap(2)
+ | statusBar
+ | kpiCards
+ : new Empty();
+
+ var showingLastSavedSnapshot = !running && done == 0 && lastSavedEffective?.Rows.Count > 0;
+ var mainTableKey = showingLastSavedSnapshot ? "run-tests-last-saved" : "run-tests-live";
+
+ var table = rows.AsQueryable()
+ .ToDataTable()
+ .RefreshToken(refreshToken)
+ .Key(mainTableKey)
+ .Height(Size.Full())
+ .Hidden(r => r.Id)
+ .Header(r => r.Widget, "Widget")
+ .Header(r => r.Difficulty, "Difficulty")
+ .Header(r => r.Question, "Question")
+ .Header(r => r.ResultIcon, "Icon")
+ .Header(r => r.Status, "Status")
+ .Header(r => r.Time, "Time")
+ .Width(r => r.ResultIcon, Size.Px(50))
+ .AlignContent(r => r.ResultIcon, Align.Center)
+ .Width(r => r.Widget, Size.Px(140))
+ .Width(r => r.Question, Size.Px(400))
+ .Width(r => r.Difficulty, Size.Px(80))
+ .Width(r => r.Status, Size.Px(90))
+ .Width(r => r.Time, Size.Px(80))
+ .Config(config =>
+ {
+ config.AllowSorting = true;
+ config.AllowFiltering = true;
+ config.ShowSearch = true;
+ config.ShowIndexColumn = true;
+ });
+
+
+ // Default Vertical gap is 4 (1rem) between every child; avoid empty Text.Muted placeholders that
+ // still consume gaps. Tight coupling between the metric/header block and the DataTable: gap 0 here.
+ var runPageTop = Layout.Vertical().Gap(2)
+ | controls
+ | historyOrActiveRunPanel;
+
+ var runPageLayout = Layout.Vertical().Height(Size.Full()).Gap(2)
+ | runPageTop
+ | afterHistoryPanel
+ | table;
+
+ object? versionExistsDialog = runVersionExistsDialogOpen.Value
+ ? new Dialog(
+ onClose: _ => runVersionExistsDialogOpen.Set(false),
+ header: new DialogHeader("This Ivy version is already in the database"),
+ body: new DialogBody(
+ Text.Block(
+ $"A completed run for \"{EffectiveIvyVersionForMcp(ivyVersion.Value.Trim(), mcpEnvironment.Value)}\" is already stored. "
+ + "You can run tests locally without saving, replace the stored run with new results, or cancel.")),
+ footer: new DialogFooter(
+ new Button("Cancel")
+ .Variant(ButtonVariant.Outline)
+ .OnClick(_ => runVersionExistsDialogOpen.Set(false)),
+ new Button("Run without saving")
+ .OnClick(async _ =>
+ {
+ runVersionExistsDialogOpen.Set(false);
+ await BeginRunAsync(persistToDatabase: false, replaceExistingRunForVersion: false);
+ }),
+ new Button("Run and replace in database")
+ .Primary()
+ .Icon(Icons.Database)
+ .OnClick(async _ =>
+ {
+ runVersionExistsDialogOpen.Set(false);
+ await BeginRunAsync(persistToDatabase: true, replaceExistingRunForVersion: true);
+ })))
+ : null;
+
+ return versionExistsDialog != null
+ ? new Fragment(runPageLayout, versionExistsDialog)
+ : runPageLayout;
+ }
+
+ /// Four metric cards before the first response returns (zeros / placeholders).
+ private static object BuildLiveRunMetricPlaceholders(int queuedCount)
+ {
+ var hint = queuedCount > 0 ? $"{queuedCount} in queue" : "Starting…";
+ return Layout.Grid().Columns(4).Height(Size.Fit())
+ | new Card(
+ Layout.Vertical()
+ | Text.H3("0%")
+ | Text.Block(hint).Muted()
+ ).Title("Answer rate").Icon(Icons.CircleCheck)
+ | new Card(
+ Layout.Vertical()
+ | Text.H3("0")
+ | Text.Block("no answer").Muted()
+ ).Title("No answer").Icon(Icons.Ban)
+ | new Card(
+ Layout.Vertical()
+ | Text.H3("0")
+ | Text.Block("failed / error").Muted()
+ ).Title("Errors").Icon(Icons.CircleX)
+ | new Card(
+ Layout.Vertical()
+ | Text.H3("—")
+ | Text.Block("waiting for timings…").Muted()
+ ).Title("Avg response").Icon(Icons.Timer);
+ }
+
+ /// Same four metric cards as after a live “Run All” completes.
+ private static object BuildOutcomeMetricCards(
+ int total,
+ int success,
+ int noAnswer,
+ int errors,
+ int avgMs,
+ int minMs,
+ int maxMs)
+ {
+ if (total <= 0)
+ return Text.Muted("");
+
+ var rate = Math.Round(success * 100.0 / total, 1);
+ var timingFooter = $"fastest {minMs} ms · slowest {maxMs} ms";
+
+ return Layout.Grid().Columns(4).Height(Size.Fit())
+ | new Card(
+ Layout.Vertical()
+ | Text.H3($"{rate}%")
+ | Text.Block($"{success} of {total} answered").Muted()
+ ).Title("Answer rate").Icon(Icons.CircleCheck)
+ | new Card(
+ Layout.Vertical()
+ | Text.H3($"{noAnswer}")
+ | Text.Block("no answer").Muted()
+ ).Title("No answer").Icon(Icons.Ban)
+ | new Card(
+ Layout.Vertical()
+ | Text.H3($"{errors}")
+ | Text.Block("failed / error").Muted()
+ ).Title("Errors").Icon(Icons.CircleX)
+ | new Card(
+ Layout.Vertical()
+ | Text.H3($"{avgMs} ms")
+ | Text.Block(timingFooter).Muted()
+ ).Title("Avg response").Icon(Icons.Timer);
+ }
+
+ private static object BuildLastSavedRunPanel(
+ QueryResult query,
+ LastSavedRunSummary? effective)
+ {
+ if (query.Loading && effective == null)
+ {
+ return Layout.Vertical().Gap(2)
+ | Text.Block("Loading last saved results…").Muted()
+ | TabLoadingSkeletons.RunMetricsRow();
+ }
+
+ var s = effective;
+ if (s == null)
+ {
+ return Layout.Vertical().Gap(2)
+ | Text.Block("No saved test run yet.").Muted()
+ | Text.Muted(
+ "The first completed run for each Ivy version is stored in the database. Re-runs for the same version stay in this session only until you refresh.");
+ }
+
+ var times = s.Rows.Select(r => r.ResponseTimeMs).ToList();
+ var avgSaved = times.Count > 0 ? (int)times.Average() : 0;
+ var minSaved = times.Count > 0 ? times.Min() : 0;
+ var maxSaved = times.Count > 0 ? times.Max() : 0;
+ var totalForMetrics = s.TotalQuestions > 0
+ ? s.TotalQuestions
+ : Math.Max(s.SuccessCount + s.NoAnswerCount + s.ErrorCount, 1);
+
+ var metricCards = BuildOutcomeMetricCards(
+ totalForMetrics,
+ s.SuccessCount,
+ s.NoAnswerCount,
+ s.ErrorCount,
+ avgSaved,
+ minSaved,
+ maxSaved);
+
+ if (s.Rows.Count == 0)
+ {
+ return Layout.Vertical().Gap(3)
+ | metricCards
+ | Text.Muted("No per-question rows linked to this run.");
+ }
+
+ return Layout.Vertical()
+ | metricCards;
+ }
+
+ private static List BuildQuestionRowsFromLastSaved(LastSavedRunSummary s) =>
+ s.Rows.Select(
+ (r, i) => new QuestionRow(
+ $"last-saved-{i}",
+ r.Widget,
+ r.Difficulty,
+ r.QuestionPreview,
+ r.Outcome == "answered"
+ ? Icons.CircleCheck
+ : r.Outcome == "no answer"
+ ? Icons.Ban
+ : Icons.CircleX,
+ r.Outcome,
+ $"{r.ResponseTimeMs} ms"))
+ .ToList();
+
+ private static async Task> LoadQuestionsAsync(
+ AppDbContextFactory factory,
+ string difficulty)
+ {
+ await using var ctx = factory.CreateDbContext();
+ var query = ctx.Questions.Where(q => q.IsActive);
+ if (difficulty != "all")
+ query = query.Where(q => q.Difficulty == difficulty);
+
+ var entities = await query
+ .OrderBy(q => q.Widget)
+ .ThenBy(q => q.Difficulty)
+ .ToListAsync();
+
+ return entities
+ .Select(e => new TestQuestion(e.Id.ToString(), e.Widget, e.Difficulty, e.QuestionText))
+ .ToList();
+ }
+
+ private const string IvyStagingVersionSuffix = "-staging";
+
+ ///
+ /// Staging runs store a distinct IvyVersion (e.g. 1.2.27-staging) so runs do not collide with production in the DB.
+ /// Production strips a trailing -staging suffix when switching MCP back.
+ ///
+ private static string EffectiveIvyVersionForMcp(string trimmed, string mcpEnvironment)
+ {
+ if (string.IsNullOrEmpty(trimmed)) return trimmed;
+ var useStaging = mcpEnvironment.Equals("staging", StringComparison.OrdinalIgnoreCase);
+ if (useStaging)
+ {
+ if (trimmed.EndsWith(IvyStagingVersionSuffix, StringComparison.OrdinalIgnoreCase))
+ return trimmed;
+ return trimmed + IvyStagingVersionSuffix;
+ }
+
+ if (trimmed.EndsWith(IvyStagingVersionSuffix, StringComparison.OrdinalIgnoreCase))
+ return trimmed[..^IvyStagingVersionSuffix.Length].TrimEnd('-').Trim();
+ return trimmed;
+ }
+
+ private static void SyncIvyVersionForMcpEnvironment(IState ivyVersion, string mcpEnvironment)
+ {
+ var raw = ivyVersion.Value.Trim();
+ if (string.IsNullOrEmpty(raw)) return;
+ var next = EffectiveIvyVersionForMcp(raw, mcpEnvironment);
+ if (!string.Equals(ivyVersion.Value.Trim(), next, StringComparison.Ordinal))
+ ivyVersion.Set(next);
+ }
+
+ private static async Task RunExistsAsync(AppDbContextFactory factory, string ivyVersion)
+ {
+ await using var ctx = factory.CreateDbContext();
+ return await ctx.TestRuns.AnyAsync(r => r.IvyVersion == ivyVersion);
+ }
+
+ ///
+ /// One row per question in order (fills gaps if the bag is short).
+ ///
+ private static List OrderResultsLikeSnapshot(
+ IReadOnlyList snapshot,
+ List bag)
+ {
+ var byId = bag
+ .GroupBy(r => r.Question.Id)
+ .ToDictionary(g => g.Key, g => g.First(), StringComparer.Ordinal);
+ return snapshot
+ .Select(q => byId.TryGetValue(q.Id, out var r)
+ ? r
+ : new QuestionRun(q, "error", 0, 0, ""))
+ .ToList();
+ }
+
+ ///
+ /// Single transaction: create run + insert every test result, or roll back (no orphan run / partial rows).
+ ///
+ private static async Task PersistNewRunAsync(
+ AppDbContextFactory factory,
+ string ivyVersion,
+ IReadOnlyList snapshot,
+ List ordered,
+ DateTime startedAtUtc,
+ string mcpEnvironment,
+ string difficultyFilter,
+ string concurrency,
+ bool replaceExistingRunForVersion)
+ {
+ if (ordered.Count != snapshot.Count)
+ return false;
+
+ await using var ctx = factory.CreateDbContext();
+ await using var tx = await ctx.Database.BeginTransactionAsync();
+ try
+ {
+ if (replaceExistingRunForVersion)
+ {
+ var oldRuns = await ctx.TestRuns.Where(r => r.IvyVersion == ivyVersion).ToListAsync();
+ if (oldRuns.Count > 0)
+ {
+ ctx.TestRuns.RemoveRange(oldRuns);
+ await ctx.SaveChangesAsync();
+ }
+ }
+
+ var run = new TestRunEntity
+ {
+ IvyVersion = ivyVersion,
+ Environment = mcpEnvironment.Trim().ToLowerInvariant() is "staging" ? "staging" : "production",
+ DifficultyFilter = string.IsNullOrEmpty(difficultyFilter) ? "all" : difficultyFilter,
+ Concurrency = concurrency ?? "",
+ TotalQuestions = snapshot.Count,
+ StartedAt = startedAtUtc,
+ SuccessCount = ordered.Count(r => r.Status == "success"),
+ NoAnswerCount = ordered.Count(r => r.Status == "no_answer"),
+ ErrorCount = ordered.Count(r => r.Status == "error"),
+ CompletedAt = DateTime.UtcNow
+ };
+ ctx.TestRuns.Add(run);
+
+ var rows = new List(ordered.Count);
+ foreach (var result in ordered)
+ {
+ if (!Guid.TryParse(result.Question.Id, out var questionId))
+ throw new InvalidOperationException($"Invalid question id: {result.Question.Id}");
+
+ rows.Add(new TestResultEntity
+ {
+ TestRunId = run.Id,
+ QuestionId = questionId,
+ ResponseText = result.AnswerText ?? "",
+ ResponseTimeMs = result.ResponseTimeMs,
+ IsSuccess = result.Status == "success",
+ HttpStatus = result.HttpStatus,
+ ErrorMessage = result.Status == "error" ? result.AnswerText : null
+ });
+ }
+
+ ctx.TestResults.AddRange(rows);
+ await ctx.SaveChangesAsync();
+ await tx.CommitAsync();
+ return true;
+ }
+ catch
+ {
+ await tx.RollbackAsync();
+ return false;
+ }
+ }
+
+ private static string ToStatusLabel(string status) => status switch
+ {
+ "success" => "answered",
+ "no_answer" => "no answer",
+ "error" => "error",
+ _ => status.Replace('_', ' ')
+ };
+
+ private static string PreviewQuestionText(string text, int maxChars)
+ {
+ if (string.IsNullOrEmpty(text) || text.Length <= maxChars)
+ return text;
+ return text[..maxChars] + "…";
+ }
+
+ private static async Task LoadLastSavedRunAsync(
+ AppDbContextFactory factory,
+ CancellationToken ct)
+ {
+ await using var ctx = factory.CreateDbContext();
+ var run = await ctx.TestRuns.AsNoTracking()
+ .OrderByDescending(r => r.CompletedAt ?? r.StartedAt)
+ .FirstOrDefaultAsync(ct);
+
+ if (run == null)
+ return null;
+
+ var rows = await (
+ from tr in ctx.TestResults.AsNoTracking()
+ join q in ctx.Questions.AsNoTracking() on tr.QuestionId equals q.Id
+ where tr.TestRunId == run.Id
+ orderby q.Widget, q.Difficulty, q.Id
+ select new LastSavedRunResultRow(
+ q.Widget,
+ q.Difficulty,
+ PreviewQuestionText(q.QuestionText ?? "", 120),
+ tr.IsSuccess
+ ? "answered"
+ : tr.HttpStatus == 404
+ ? "no answer"
+ : "error",
+ tr.ResponseTimeMs))
+ .ToListAsync(ct);
+
+ return new LastSavedRunSummary(
+ run.Id,
+ run.IvyVersion,
+ run.Environment,
+ string.IsNullOrEmpty(run.DifficultyFilter) ? "all" : run.DifficultyFilter,
+ run.Concurrency ?? "",
+ run.TotalQuestions,
+ run.SuccessCount,
+ run.NoAnswerCount,
+ run.ErrorCount,
+ run.StartedAt,
+ run.CompletedAt,
+ rows);
+ }
+}
+
+///
+/// Remembers Run form fields for the current server process so tab switches do not reset MCP / difficulty / version.
+///
+file static class RunTestFormPreferences
+{
+ public static string IvyVersion { get; private set; } = "";
+
+ public static string McpEnvironment { get; private set; } = "production";
+
+ public static string DifficultyFilter { get; private set; } = "all";
+
+ public static void Set(string ivy, string mcp, string diff)
+ {
+ IvyVersion = ivy ?? "";
+ McpEnvironment = string.IsNullOrEmpty(mcp) ? "production" : mcp;
+ DifficultyFilter = string.IsNullOrEmpty(diff) ? "all" : diff;
+ }
+}
diff --git a/project-demos/ivy-ask-statistics/Apps/TabLoadingSkeletons.cs b/project-demos/ivy-ask-statistics/Apps/TabLoadingSkeletons.cs
new file mode 100644
index 00000000..8e6d7185
--- /dev/null
+++ b/project-demos/ivy-ask-statistics/Apps/TabLoadingSkeletons.cs
@@ -0,0 +1,106 @@
+namespace IvyAskStatistics.Apps;
+
+/// Lightweight placeholders while queries load. Nested Height(Full) on skeletons was removed
+/// because it can thrash layout and look like loading never finishes.
+internal static class TabLoadingSkeletons
+{
+ public static object Dashboard()
+ {
+ // Mirrors : 5 KPI cards → 3 version-compare charts → 4 env charts → Test runs table.
+ static object KpiCard(string title, Icons icon) =>
+ new Card(
+ Layout.Vertical().AlignContent(Align.Center).Gap(1)
+ | (Layout.Horizontal().AlignContent(Align.Center).Gap(1)
+ | new Skeleton().Height(Size.Units(9)).Width(Size.Px(56))
+ | new Skeleton().Height(Size.Units(5)).Width(Size.Px(40)))
+ ).Title(title).Icon(icon);
+
+ var kpiRow = Layout.Grid().Columns(5).Height(Size.Fit())
+ | KpiCard("Answer success", Icons.CircleCheck)
+ | KpiCard("Avg latency", Icons.Timer)
+ | KpiCard("No answer + errors", Icons.CircleX)
+ | KpiCard("Weakest widget", Icons.Ban)
+ | KpiCard("Ivy version", Icons.Tag);
+
+ static object VersionChartCard(string title) =>
+ new Card(
+ Layout.Vertical().Gap(3)
+ | new Skeleton().Height(Size.Units(5)).Width(Size.Px(180))
+ | new Skeleton().Height(Size.Units(48)).Width(Size.Fraction(1f)))
+ .Title(title)
+ .Height(Size.Units(70));
+
+ var versionChartsRow = Layout.Grid().Columns(3).Height(Size.Fit())
+ | VersionChartCard("Success rate · production vs staging")
+ | VersionChartCard("Avg response · production vs staging")
+ | VersionChartCard("Outcomes · production vs staging");
+
+ static object DetailChartCard(string title) =>
+ new Card(
+ Layout.Vertical().Gap(3)
+ | new Skeleton().Height(Size.Units(5)).Width(Size.Px(200))
+ | new Skeleton().Height(Size.Units(48)).Width(Size.Fraction(1f)))
+ .Title(title)
+ .Height(Size.Units(70));
+
+ const string envLabel = "Production";
+ var detailChartsRow = Layout.Grid().Columns(4).Height(Size.Fit())
+ | DetailChartCard($"Worst widgets — rate % ({envLabel})")
+ | DetailChartCard($"Latency by widget — avg / max ({envLabel})")
+ | DetailChartCard($"Results by difficulty ({envLabel})")
+ | DetailChartCard($"Result mix ({envLabel})");
+
+ var testRunsCard = new Card(
+ Layout.Vertical().Gap(2)
+ | new Skeleton().Height(Size.Units(5)).Width(Size.Fraction(1f))
+ | new Skeleton().Height(Size.Units(5)).Width(Size.Fraction(1f))
+ | new Skeleton().Height(Size.Units(5)).Width(Size.Fraction(1f))
+ | new Skeleton().Height(Size.Units(115)).Width(Size.Fraction(1f)))
+ .Title("Test runs");
+
+ return Layout.Vertical().Height(Size.Full())
+ | kpiRow
+ | versionChartsRow
+ | detailChartsRow
+ | (Layout.Vertical() | testRunsCard);
+ }
+
+ /// Four KPI cards while loads the last saved run (titles match BuildOutcomeMetricCards).
+ public static object RunMetricsRow()
+ {
+ static object KpiCard(string title, Icons icon) =>
+ new Card(
+ Layout.Vertical().AlignContent(Align.Center).Gap(2)
+ | new Skeleton().Height(Size.Units(9)).Width(Size.Px(72))
+ | new Skeleton().Height(Size.Units(4)).Width(Size.Px(140)))
+ .Title(title)
+ .Icon(icon);
+
+ return Layout.Grid().Columns(4).Height(Size.Fit())
+ | KpiCard("Answer rate", Icons.CircleCheck)
+ | KpiCard("No answer", Icons.Ban)
+ | KpiCard("Errors", Icons.CircleX)
+ | KpiCard("Avg response", Icons.Timer);
+ }
+
+ public static object RunTab() =>
+ DataTableToolbarAndBody(tableBodyHeightUnits: 280);
+
+ public static object QuestionsTab() =>
+ DataTableToolbarAndBody(tableBodyHeightUnits: 260);
+
+ /// Filter-sized square + single block for the DataTable body (, ).
+ private static object DataTableToolbarAndBody(int tableBodyHeightUnits) =>
+ Layout.Vertical().Height(Size.Full()).Gap(3)
+ | new Skeleton().Height(Size.Px(36)).Width(Size.Px(36))
+ | new Skeleton().Height(Size.Units(tableBodyHeightUnits)).Width(Size.Fraction(1f));
+
+ public static object DialogTable()
+ {
+ return Layout.Vertical().Width(Size.Fraction(1f)).Height(Size.Fit())
+ | new Skeleton().Height(Size.Units(10)).Width(Size.Fraction(1f))
+ | new Skeleton().Height(Size.Units(10)).Width(Size.Fraction(1f))
+ | new Skeleton().Height(Size.Units(10)).Width(Size.Fraction(1f))
+ | new Skeleton().Height(Size.Units(80)).Width(Size.Fraction(1f));
+ }
+}
diff --git a/project-demos/ivy-ask-statistics/Apps/TestRunResultsDialog.cs b/project-demos/ivy-ask-statistics/Apps/TestRunResultsDialog.cs
new file mode 100644
index 00000000..6c21164d
--- /dev/null
+++ b/project-demos/ivy-ask-statistics/Apps/TestRunResultsDialog.cs
@@ -0,0 +1,197 @@
+namespace IvyAskStatistics.Apps;
+
+/// Modal listing all test results for a single test run, with edit/delete row actions.
+internal sealed class TestRunResultsDialog(
+ IState isOpen,
+ Guid runId,
+ IState editSheetOpen,
+ IState editQuestionId,
+ IState editPreviewResultId) : ViewBase
+{
+ public override object? Build()
+ {
+ var factory = UseService();
+ var client = UseService();
+ var queryService = UseService();
+ var refreshToken = UseRefreshToken();
+
+ var (alertView, showAlert) = UseAlert();
+
+ var resultsQuery = UseQuery(
+ key: $"run-results-{runId}",
+ fetcher: async (_, ct) =>
+ {
+ var payload = await LoadAsync(factory, runId, ct);
+ refreshToken.Refresh();
+ return payload;
+ },
+ options: new QueryOptions { KeepPrevious = true },
+ tags: [("run-results", runId.ToString())]);
+
+ var firstLoad = resultsQuery.Loading && resultsQuery.Value == null;
+ var rows = resultsQuery.Value?.Results ?? [];
+ var runInfo = resultsQuery.Value?.RunInfo;
+
+ void Close() => isOpen.Set(false);
+
+ async Task DeleteAsync(Guid resultId)
+ {
+ try
+ {
+ await using var ctx = factory.CreateDbContext();
+ var entity = await ctx.TestResults.FirstOrDefaultAsync(r => r.Id == resultId);
+ if (entity != null)
+ {
+ ctx.TestResults.Remove(entity);
+ await ctx.SaveChangesAsync();
+
+ var run = await ctx.TestRuns.FirstOrDefaultAsync(r => r.Id == runId);
+ if (run != null)
+ {
+ var remaining = await ctx.TestResults
+ .Where(r => r.TestRunId == runId)
+ .ToListAsync();
+ run.TotalQuestions = remaining.Count;
+ run.SuccessCount = remaining.Count(r => r.IsSuccess);
+ run.NoAnswerCount = remaining.Count(r => !r.IsSuccess && r.HttpStatus == 404);
+ run.ErrorCount = remaining.Count(r => !r.IsSuccess && r.HttpStatus != 404);
+ await ctx.SaveChangesAsync();
+ }
+ }
+
+ var updated = rows.Where(r => r.ResultId != resultId).ToList();
+ resultsQuery.Mutator.Mutate(new RunResultsPayload(runInfo, updated), revalidate: false);
+ refreshToken.Refresh();
+ queryService.RevalidateByTag("dashboard-stats");
+ }
+ catch (Exception ex)
+ {
+ client.Toast($"Error: {ex.Message}");
+ }
+ }
+
+ object body;
+ if (firstLoad)
+ body = TabLoadingSkeletons.DialogTable();
+ else if (rows.Count == 0)
+ body = new Callout("No results found for this run.", variant: CalloutVariant.Info);
+ else
+ {
+ body = rows.AsQueryable()
+ .ToDataTable(r => r.ResultId)
+ .Key($"run-results-tbl-{runId}")
+ .Height(Size.Units(120))
+ .RefreshToken(refreshToken)
+ .Header(r => r.Widget, "Widget")
+ .Header(r => r.Difficulty, "Difficulty")
+ .Header(r => r.Question, "Question")
+ .Header(r => r.Status, "Status")
+ .Header(r => r.Response, "Response")
+ .Header(r => r.ResponseTimeMs, "Time (ms)")
+ .Width(r => r.Widget, Size.Px(130))
+ .Width(r => r.Difficulty, Size.Px(80))
+ .Width(r => r.Status, Size.Px(90))
+ .Width(r => r.ResponseTimeMs, Size.Px(80))
+ .Width(r => r.Response, Size.Px(300))
+ .Hidden(r => r.ResultId)
+ .Hidden(r => r.QuestionId)
+ .AlignContent(r => r.ResponseTimeMs, Align.Left)
+ .RowActions(
+ MenuItem.Default(Icons.Pencil, "edit").Label("Edit question").Tag("edit"),
+ MenuItem.Default(Icons.Trash2, "delete").Label("Delete result").Tag("delete"))
+ .OnRowAction(e =>
+ {
+ var args = e.Value;
+ var tag = args?.Tag?.ToString();
+ if (!Guid.TryParse(args?.Id?.ToString(), out var resultId))
+ return ValueTask.CompletedTask;
+
+ var row = rows.FirstOrDefault(r => r.ResultId == resultId);
+ if (row == null) return ValueTask.CompletedTask;
+
+ if (tag == "edit")
+ {
+ editQuestionId.Set(row.QuestionId);
+ editPreviewResultId.Set(row.ResultId);
+ editSheetOpen.Set(true);
+ }
+ else if (tag == "delete")
+ {
+ var preview = row.Question.Length > 60 ? row.Question[..60] + "…" : row.Question;
+ showAlert(
+ $"Delete this result?\n\n\"{preview}\"",
+ async result =>
+ {
+ if (!result.IsOk()) return;
+ await DeleteAsync(resultId);
+ },
+ "Delete result",
+ AlertButtonSet.OkCancel);
+ }
+
+ return ValueTask.CompletedTask;
+ })
+ .Config(c =>
+ {
+ c.AllowSorting = true;
+ c.AllowFiltering = true;
+ c.ShowSearch = true;
+ c.ShowIndexColumn = false;
+ });
+ }
+
+ var title = firstLoad
+ ? "Loading results…"
+ : runInfo != null
+ ? $"Results — {runInfo.IvyVersion} · {runInfo.Environment} ({rows.Count})"
+ : $"Results ({rows.Count})";
+
+ return new Fragment(
+ alertView,
+ new Dialog(
+ onClose: _ => Close(),
+ header: new DialogHeader(title),
+ body: new DialogBody(body),
+ footer: new DialogFooter(new Button("Close").OnClick(_ => Close())))
+ .Width(Size.Units(320)));
+ }
+
+ private static async Task LoadAsync(
+ AppDbContextFactory factory, Guid runId, CancellationToken ct)
+ {
+ await using var ctx = factory.CreateDbContext();
+
+ var run = await ctx.TestRuns.AsNoTracking().FirstOrDefaultAsync(r => r.Id == runId, ct);
+ var runInfo = run == null ? null : new RunInfo(run.IvyVersion ?? "", run.Environment ?? "production");
+
+ var results = await ctx.TestResults.AsNoTracking()
+ .Include(r => r.Question)
+ .Where(r => r.TestRunId == runId)
+ .OrderBy(r => r.Question.Widget)
+ .ThenBy(r => r.Question.Difficulty)
+ .Select(r => new RunResultRow(
+ r.Id,
+ r.QuestionId,
+ r.Question.Widget,
+ r.Question.Difficulty,
+ r.Question.QuestionText,
+ r.IsSuccess ? "answered" : r.HttpStatus == 404 ? "no answer" : "error",
+ r.ResponseText,
+ r.ResponseTimeMs))
+ .ToListAsync(ct);
+
+ return new RunResultsPayload(runInfo, results);
+ }
+}
+
+internal record RunInfo(string IvyVersion, string Environment);
+internal record RunResultsPayload(RunInfo? RunInfo, List Results);
+internal record RunResultRow(
+ Guid ResultId,
+ Guid QuestionId,
+ string Widget,
+ string Difficulty,
+ string Question,
+ string Status,
+ string Response,
+ int ResponseTimeMs);
diff --git a/project-demos/ivy-ask-statistics/Apps/WidgetQuestionsDialog.cs b/project-demos/ivy-ask-statistics/Apps/WidgetQuestionsDialog.cs
new file mode 100644
index 00000000..9ee1146b
--- /dev/null
+++ b/project-demos/ivy-ask-statistics/Apps/WidgetQuestionsDialog.cs
@@ -0,0 +1,164 @@
+namespace IvyAskStatistics.Apps;
+
+/// Modal listing all DB questions for a single widget.
+internal sealed class WidgetQuestionsDialog(
+ IState isOpen,
+ string widgetName,
+ IState editSheetOpen,
+ IState editQuestionId,
+ IState editPreviewResultId) : ViewBase
+{
+ public override object? Build()
+ {
+ var factory = UseService();
+ var client = UseService();
+ var queryService = UseService();
+ var refreshToken = UseRefreshToken();
+
+ var (alertView, showAlert) = UseAlert();
+
+ var tableQuery = UseQuery, string>(
+ key: widgetName,
+ fetcher: async (name, ct) =>
+ {
+ var result = await LoadQuestionsAsync(factory, name, ct);
+ refreshToken.Refresh();
+ return result;
+ },
+ options: new QueryOptions { KeepPrevious = true },
+ tags: [("widget-questions", widgetName)]);
+
+ var firstLoad = tableQuery.Loading && tableQuery.Value == null;
+ var rows = tableQuery.Value ?? [];
+
+ void Close() => isOpen.Set(false);
+
+ async Task DeleteAsync(Guid id)
+ {
+ try
+ {
+ await using var ctx = factory.CreateDbContext();
+ var entity = await ctx.Questions.FirstOrDefaultAsync(q => q.Id == id);
+ if (entity != null)
+ {
+ ctx.Questions.Remove(entity);
+ await ctx.SaveChangesAsync();
+ }
+ var updated = rows.Where(r => r.Id != id).ToList();
+ tableQuery.Mutator.Mutate(updated, revalidate: false);
+ refreshToken.Refresh();
+ queryService.RevalidateByTag("widget-summary");
+ queryService.RevalidateByTag(RunApp.TestQuestionsQueryTag);
+ }
+ catch (Exception ex)
+ {
+ client.Toast($"Error: {ex.Message}");
+ }
+ }
+
+ object body;
+ if (firstLoad)
+ body = TabLoadingSkeletons.DialogTable();
+ else if (rows.Count == 0)
+ {
+ body = new Callout(
+ $"No questions in the database for \"{widgetName}\".",
+ variant: CalloutVariant.Info);
+ }
+ else
+ {
+ body = rows.AsQueryable()
+ .ToDataTable(r => r.Id)
+ .Key($"widget-questions-{widgetName}")
+ .Height(Size.Units(120))
+ .RefreshToken(refreshToken)
+ .Header(r => r.Difficulty, "Difficulty")
+ .Header(r => r.Category, "Category")
+ .Header(r => r.QuestionText, "Question")
+ .Header(r => r.Source, "Source")
+ .Header(r => r.CreatedAt, "Created")
+ .Width(r => r.Difficulty, Size.Px(80))
+ .Width(r => r.QuestionText, Size.Px(340))
+ .Width(r => r.Source, Size.Px(100))
+ .Width(r => r.CreatedAt, Size.Px(170))
+ .Hidden(r => r.Id)
+ .RowActions(
+ MenuItem.Default(Icons.Pencil, "edit").Label("Edit").Tag("edit"),
+ MenuItem.Default(Icons.Trash2, "delete").Label("Delete").Tag("delete"))
+ .OnRowAction(e =>
+ {
+ var args = e.Value;
+ var tag = args?.Tag?.ToString();
+ if (!Guid.TryParse(args?.Id?.ToString(), out var id)) return ValueTask.CompletedTask;
+
+ if (tag == "edit")
+ {
+ editPreviewResultId.Set(null);
+ editQuestionId.Set(id);
+ editSheetOpen.Set(true);
+ }
+ else if (tag == "delete")
+ {
+ var text = rows.FirstOrDefault(r => r.Id == id)?.QuestionText ?? "";
+ var preview = text.Length > 60 ? text[..60] + "…" : text;
+ showAlert(
+ $"Delete this question?\n\n\"{preview}\"",
+ async result =>
+ {
+ if (!result.IsOk()) return;
+ await DeleteAsync(id);
+ },
+ "Delete question",
+ AlertButtonSet.OkCancel);
+ }
+
+ return ValueTask.CompletedTask;
+ })
+ .Config(c =>
+ {
+ c.AllowSorting = true;
+ c.AllowFiltering = true;
+ c.ShowSearch = true;
+ c.ShowIndexColumn = false;
+ });
+ }
+
+ var title = firstLoad || rows.Count == 0
+ ? $"Questions — {widgetName}"
+ : $"Questions — {widgetName} ({rows.Count})";
+
+ var footer = new DialogFooter(new Button("Close").OnClick(_ => Close()));
+
+ return new Fragment(
+ alertView,
+ new Dialog(
+ onClose: _ => Close(),
+ header: new DialogHeader(title),
+ body: new DialogBody(body),
+ footer: footer)
+ .Width(Size.Units(240)));
+ }
+
+ private static async Task> LoadQuestionsAsync(
+ AppDbContextFactory factory,
+ string name,
+ CancellationToken ct)
+ {
+ await using var ctx = factory.CreateDbContext();
+
+ return await ctx.Questions
+ .AsNoTracking()
+ .Where(q => q.Widget == name)
+ .OrderBy(q => q.Difficulty)
+ .ThenBy(q => q.Category)
+ .ThenBy(q => q.CreatedAt)
+ .Select(q => new QuestionDetailRow(
+ q.Id,
+ q.Difficulty,
+ q.Category,
+ q.QuestionText ?? "",
+ q.Source,
+ q.CreatedAt.ToLocalTime().ToString("dd MMM yyyy, HH:mm", CultureInfo.CurrentCulture)))
+ .ToListAsync(ct);
+ }
+}
diff --git a/project-demos/ivy-ask-statistics/Connections/AppConnection.cs b/project-demos/ivy-ask-statistics/Connections/AppConnection.cs
new file mode 100644
index 00000000..dedaede5
--- /dev/null
+++ b/project-demos/ivy-ask-statistics/Connections/AppConnection.cs
@@ -0,0 +1,58 @@
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Configuration;
+
+namespace IvyAskStatistics.Connections;
+
+public class AppConnection : IConnection
+{
+ public string GetName() => "IvyAskDb";
+
+ public string GetNamespace() => typeof(AppConnection).Namespace!;
+
+ public string GetConnectionType() => "EntityFramework.PostgreSQL";
+
+ public string GetContext(string connectionPath) => "";
+
+ public ConnectionEntity[] GetEntities()
+ {
+ return typeof(AppDbContext)
+ .GetProperties()
+ .Where(p => p.PropertyType.IsGenericType &&
+ p.PropertyType.GetGenericTypeDefinition() == typeof(DbSet<>))
+ .Select(p => new ConnectionEntity(
+ p.PropertyType.GenericTypeArguments[0].Name,
+ p.Name))
+ .ToArray();
+ }
+
+ public Task<(bool ok, string? message)> TestConnection(IConfiguration configuration)
+ {
+ try
+ {
+ var cs = configuration["DB_CONNECTION_STRING"];
+ if (string.IsNullOrWhiteSpace(cs))
+ return Task.FromResult<(bool, string?)>((false, "DB_CONNECTION_STRING not set in user-secrets"));
+
+ var options = new DbContextOptionsBuilder().UseNpgsql(cs).Options;
+ using var ctx = new AppDbContext(options);
+ var canConnect = ctx.Database.CanConnect();
+ return Task.FromResult<(bool, string?)>(canConnect
+ ? (true, null)
+ : (false, "Cannot connect to database"));
+ }
+ catch (Exception ex)
+ {
+ return Task.FromResult<(bool, string?)>((false, ex.Message));
+ }
+ }
+
+ public void RegisterServices(Server server)
+ {
+ server.Services.AddSingleton();
+ }
+
+ public Secret[] GetSecrets()
+ {
+ return [new Secret("DB_CONNECTION_STRING", "PostgreSQL connection string (Supabase)")];
+ }
+}
diff --git a/project-demos/ivy-ask-statistics/Connections/AppDbContext.cs b/project-demos/ivy-ask-statistics/Connections/AppDbContext.cs
new file mode 100644
index 00000000..8ddc7ecd
--- /dev/null
+++ b/project-demos/ivy-ask-statistics/Connections/AppDbContext.cs
@@ -0,0 +1,39 @@
+using Microsoft.EntityFrameworkCore;
+
+namespace IvyAskStatistics.Connections;
+
+public class AppDbContext(DbContextOptions options) : DbContext(options)
+{
+ public DbSet Questions { get; set; }
+ public DbSet TestRuns { get; set; }
+ public DbSet TestResults { get; set; }
+
+ protected override void OnModelCreating(ModelBuilder modelBuilder)
+ {
+ modelBuilder.Entity(e =>
+ {
+ e.HasOne(r => r.TestRun)
+ .WithMany(tr => tr.Results)
+ .HasForeignKey(r => r.TestRunId)
+ .OnDelete(DeleteBehavior.Cascade);
+
+ e.HasOne(r => r.Question)
+ .WithMany(q => q.TestResults)
+ .HasForeignKey(r => r.QuestionId)
+ .OnDelete(DeleteBehavior.Cascade);
+
+ e.HasIndex(r => r.TestRunId);
+ e.HasIndex(r => r.QuestionId);
+ });
+
+ modelBuilder.Entity(e =>
+ {
+ e.HasIndex(r => r.IvyVersion);
+ });
+
+ modelBuilder.Entity(e =>
+ {
+ e.HasIndex(q => q.IsActive);
+ });
+ }
+}
diff --git a/project-demos/ivy-ask-statistics/Connections/AppDbContextFactory.cs b/project-demos/ivy-ask-statistics/Connections/AppDbContextFactory.cs
new file mode 100644
index 00000000..09982f69
--- /dev/null
+++ b/project-demos/ivy-ask-statistics/Connections/AppDbContextFactory.cs
@@ -0,0 +1,99 @@
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Configuration;
+
+namespace IvyAskStatistics.Connections;
+
+public sealed class AppDbContextFactory : IDbContextFactory
+{
+ private readonly IConfiguration _config;
+ private static bool _initialized;
+ private static readonly SemaphoreSlim _initLock = new(1, 1);
+
+ public AppDbContextFactory(IConfiguration config)
+ {
+ _config = config;
+ EnsureInitialized();
+ }
+
+ public AppDbContext CreateDbContext()
+ {
+ var ctx = new AppDbContext(BuildOptions());
+ EnsureInitialized(ctx);
+ return ctx;
+ }
+
+ private void EnsureInitialized()
+ {
+ using var ctx = new AppDbContext(BuildOptions());
+ EnsureInitialized(ctx);
+ }
+
+ private void EnsureInitialized(AppDbContext ctx)
+ {
+ if (_initialized) return;
+ _initLock.Wait();
+ try
+ {
+ if (_initialized) return;
+ // EnsureCreated() does nothing when the database already exists (e.g. Supabase).
+ // Use explicit CREATE TABLE IF NOT EXISTS instead.
+ ctx.Database.ExecuteSqlRaw("""
+ CREATE TABLE IF NOT EXISTS ivy_ask_questions (
+ "Id" UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+ "Widget" VARCHAR(100) NOT NULL,
+ "Category" VARCHAR(100) NOT NULL DEFAULT '',
+ "Difficulty" VARCHAR(10) NOT NULL,
+ "QuestionText" TEXT NOT NULL,
+ "Source" VARCHAR(20) NOT NULL DEFAULT 'manual',
+ "CreatedAt" TIMESTAMPTZ NOT NULL DEFAULT NOW()
+ );
+ ALTER TABLE ivy_ask_questions ADD COLUMN IF NOT EXISTS "IsActive" BOOLEAN NOT NULL DEFAULT TRUE;
+
+ CREATE TABLE IF NOT EXISTS ivy_ask_test_runs (
+ "Id" UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+ "IvyVersion" VARCHAR(50) NOT NULL DEFAULT '',
+ "Environment" VARCHAR(50) NOT NULL DEFAULT 'production',
+ "TotalQuestions" INTEGER NOT NULL DEFAULT 0,
+ "SuccessCount" INTEGER NOT NULL DEFAULT 0,
+ "NoAnswerCount" INTEGER NOT NULL DEFAULT 0,
+ "ErrorCount" INTEGER NOT NULL DEFAULT 0,
+ "StartedAt" TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+ "CompletedAt" TIMESTAMPTZ
+ );
+ CREATE INDEX IF NOT EXISTS ix_test_runs_ivy_version ON ivy_ask_test_runs ("IvyVersion");
+ ALTER TABLE ivy_ask_test_runs ADD COLUMN IF NOT EXISTS "DifficultyFilter" VARCHAR(20) NOT NULL DEFAULT 'all';
+ ALTER TABLE ivy_ask_test_runs ADD COLUMN IF NOT EXISTS "Concurrency" VARCHAR(10) NOT NULL DEFAULT '';
+
+ CREATE TABLE IF NOT EXISTS ivy_ask_test_results (
+ "Id" UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+ "TestRunId" UUID NOT NULL REFERENCES ivy_ask_test_runs("Id") ON DELETE CASCADE,
+ "QuestionId" UUID NOT NULL REFERENCES ivy_ask_questions("Id") ON DELETE CASCADE,
+ "ResponseText" TEXT NOT NULL DEFAULT '',
+ "ResponseTimeMs" INTEGER NOT NULL DEFAULT 0,
+ "IsSuccess" BOOLEAN NOT NULL DEFAULT FALSE,
+ "HttpStatus" INTEGER NOT NULL DEFAULT 0,
+ "ErrorMessage" VARCHAR(500),
+ "CreatedAt" TIMESTAMPTZ NOT NULL DEFAULT NOW()
+ );
+ CREATE INDEX IF NOT EXISTS ix_test_results_run_id ON ivy_ask_test_results ("TestRunId");
+ CREATE INDEX IF NOT EXISTS ix_test_results_question_id ON ivy_ask_test_results ("QuestionId");
+ """);
+ _initialized = true;
+ }
+ finally
+ {
+ _initLock.Release();
+ }
+ }
+
+ private DbContextOptions BuildOptions()
+ {
+ var cs = _config["DB_CONNECTION_STRING"]
+ ?? throw new InvalidOperationException(
+ "DB_CONNECTION_STRING not set. Run: dotnet user-secrets set \"DB_CONNECTION_STRING\" \"\"");
+
+ return new DbContextOptionsBuilder()
+ .UseNpgsql(cs)
+ .Options;
+ }
+}
diff --git a/project-demos/ivy-ask-statistics/Connections/Auth/BasicAuthConnection.cs b/project-demos/ivy-ask-statistics/Connections/Auth/BasicAuthConnection.cs
new file mode 100644
index 00000000..ec95e06c
--- /dev/null
+++ b/project-demos/ivy-ask-statistics/Connections/Auth/BasicAuthConnection.cs
@@ -0,0 +1,37 @@
+using Ivy;
+using Microsoft.Extensions.Configuration;
+
+namespace IvyAskStatistics.Connections.Auth;
+
+public class BasicAuthConnection : IConnection, IHaveSecrets
+{
+ public string GetContext(string connectionPath) => string.Empty;
+
+ public string GetName() => "BasicAuth";
+
+ public string GetNamespace() => typeof(BasicAuthConnection).Namespace ?? "";
+
+ public string GetConnectionType() => "Auth";
+
+ public ConnectionEntity[] GetEntities() => [];
+
+ public void RegisterServices(Server server)
+ {
+ server.UseAuth();
+ }
+
+ public Secret[] GetSecrets() =>
+ [
+ new("BasicAuth:Users"),
+ new("BasicAuth:HashSecret"),
+ new("BasicAuth:JwtSecret"),
+ new("BasicAuth:JwtIssuer"),
+ new("BasicAuth:JwtAudience")
+ ];
+
+ public async Task<(bool ok, string? message)> TestConnection(IConfiguration config)
+ {
+ await Task.CompletedTask;
+ return (true, "Basic Auth configured");
+ }
+}
diff --git a/project-demos/ivy-ask-statistics/Connections/QuestionEntity.cs b/project-demos/ivy-ask-statistics/Connections/QuestionEntity.cs
new file mode 100644
index 00000000..35b8e716
--- /dev/null
+++ b/project-demos/ivy-ask-statistics/Connections/QuestionEntity.cs
@@ -0,0 +1,30 @@
+using System.ComponentModel.DataAnnotations;
+using System.ComponentModel.DataAnnotations.Schema;
+
+namespace IvyAskStatistics.Connections;
+
+[Table("ivy_ask_questions")]
+public class QuestionEntity
+{
+ public Guid Id { get; set; } = Guid.NewGuid();
+
+ [MaxLength(100)]
+ public string Widget { get; set; } = "";
+
+ [MaxLength(100)]
+ public string Category { get; set; } = "";
+
+ [MaxLength(10)]
+ public string Difficulty { get; set; } = "";
+
+ public string QuestionText { get; set; } = "";
+
+ [MaxLength(20)]
+ public string Source { get; set; } = "manual";
+
+ public bool IsActive { get; set; } = true;
+
+ public DateTime CreatedAt { get; set; } = DateTime.UtcNow;
+
+ public List TestResults { get; set; } = [];
+}
diff --git a/project-demos/ivy-ask-statistics/Connections/TestRunEntity.cs b/project-demos/ivy-ask-statistics/Connections/TestRunEntity.cs
new file mode 100644
index 00000000..8426f984
--- /dev/null
+++ b/project-demos/ivy-ask-statistics/Connections/TestRunEntity.cs
@@ -0,0 +1,33 @@
+using System.ComponentModel.DataAnnotations;
+using System.ComponentModel.DataAnnotations.Schema;
+
+namespace IvyAskStatistics.Connections;
+
+[Table("ivy_ask_test_runs")]
+public class TestRunEntity
+{
+ public Guid Id { get; set; } = Guid.NewGuid();
+
+ [MaxLength(50)]
+ public string IvyVersion { get; set; } = "";
+
+ [MaxLength(50)]
+ public string Environment { get; set; } = "production";
+
+ /// Difficulty scope for the run: all, easy, medium, hard.
+ [MaxLength(20)]
+ public string DifficultyFilter { get; set; } = "all";
+
+ [MaxLength(10)]
+ public string Concurrency { get; set; } = "";
+
+ public int TotalQuestions { get; set; }
+ public int SuccessCount { get; set; }
+ public int NoAnswerCount { get; set; }
+ public int ErrorCount { get; set; }
+
+ public DateTime StartedAt { get; set; } = DateTime.UtcNow;
+ public DateTime? CompletedAt { get; set; }
+
+ public List Results { get; set; } = [];
+}
diff --git a/project-demos/ivy-ask-statistics/Dockerfile b/project-demos/ivy-ask-statistics/Dockerfile
new file mode 100644
index 00000000..2ad4e0fe
--- /dev/null
+++ b/project-demos/ivy-ask-statistics/Dockerfile
@@ -0,0 +1,34 @@
+# Base runtime image
+FROM mcr.microsoft.com/dotnet/aspnet:10.0 AS base
+WORKDIR /app
+EXPOSE 80
+
+# Build stage
+FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build
+ARG BUILD_CONFIGURATION=Release
+WORKDIR /src
+
+# Copy and restore
+COPY ["IvyAskStatistics.csproj", "./"]
+RUN dotnet restore "IvyAskStatistics.csproj"
+
+# Copy everything and build
+COPY . .
+RUN dotnet build "IvyAskStatistics.csproj" -c $BUILD_CONFIGURATION -o /app/build
+
+# Publish stage
+FROM build AS publish
+ARG BUILD_CONFIGURATION=Release
+RUN dotnet publish "IvyAskStatistics.csproj" -c $BUILD_CONFIGURATION -o /app/publish /p:UseAppHost=true
+
+# Final runtime image
+FROM base AS final
+WORKDIR /app
+COPY --from=publish /app/publish .
+
+# Set environment variables
+ENV PORT=80
+ENV ASPNETCORE_URLS="http://+:80"
+
+# Run the executable
+ENTRYPOINT ["dotnet", "./IvyAskStatistics.dll"]
diff --git a/project-demos/ivy-ask-statistics/GlobalUsings.cs b/project-demos/ivy-ask-statistics/GlobalUsings.cs
new file mode 100644
index 00000000..f89be426
--- /dev/null
+++ b/project-demos/ivy-ask-statistics/GlobalUsings.cs
@@ -0,0 +1,10 @@
+global using Ivy;
+global using System.ComponentModel.DataAnnotations;
+global using IvyAskStatistics.Models;
+global using IvyAskStatistics.Services;
+global using IvyAskStatistics.Connections;
+global using Microsoft.EntityFrameworkCore;
+global using Microsoft.Extensions.Configuration;
+global using Microsoft.Extensions.DependencyInjection;
+global using System.Globalization;
+global using System.Net;
diff --git a/project-demos/ivy-ask-statistics/IvyAskStatistics.csproj b/project-demos/ivy-ask-statistics/IvyAskStatistics.csproj
new file mode 100644
index 00000000..6977cff2
--- /dev/null
+++ b/project-demos/ivy-ask-statistics/IvyAskStatistics.csproj
@@ -0,0 +1,34 @@
+
+
+
+ Exe
+ net10.0
+ enable
+ enable
+ CS8618;CS8603;CS8602;CS8604;CS9113
+ IvyAskStatistics
+ ivy-ask-statistics-a3f2c1d4
+
+
+
+
+
+ all
+ runtime; build; native; contentfiles; analyzers; buildtransitive
+
+
+ runtime; build; native; contentfiles; analyzers; buildtransitive
+ all
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/project-demos/ivy-ask-statistics/Models/IvyWidget.cs b/project-demos/ivy-ask-statistics/Models/IvyWidget.cs
new file mode 100644
index 00000000..142f6107
--- /dev/null
+++ b/project-demos/ivy-ask-statistics/Models/IvyWidget.cs
@@ -0,0 +1,3 @@
+namespace IvyAskStatistics.Models;
+
+public record IvyWidget(string Name, string Category, string DocLink);
diff --git a/project-demos/ivy-ask-statistics/Models/QuestionDetailRow.cs b/project-demos/ivy-ask-statistics/Models/QuestionDetailRow.cs
new file mode 100644
index 00000000..6e373186
--- /dev/null
+++ b/project-demos/ivy-ask-statistics/Models/QuestionDetailRow.cs
@@ -0,0 +1,9 @@
+namespace IvyAskStatistics.Models;
+
+public record QuestionDetailRow(
+ Guid Id,
+ string Difficulty,
+ string Category,
+ string QuestionText,
+ string Source,
+ string CreatedAt);
diff --git a/project-demos/ivy-ask-statistics/Models/RunModels.cs b/project-demos/ivy-ask-statistics/Models/RunModels.cs
new file mode 100644
index 00000000..7abc66ca
--- /dev/null
+++ b/project-demos/ivy-ask-statistics/Models/RunModels.cs
@@ -0,0 +1,43 @@
+namespace IvyAskStatistics.Models;
+
+public record TestQuestion(string Id, string Widget, string Difficulty, string Question);
+
+public record QuestionRun(
+ TestQuestion Question,
+ string Status, // "success" | "no_answer" | "error"
+ int ResponseTimeMs,
+ int HttpStatus,
+ string AnswerText = "" // raw response body; empty when no_answer or error
+);
+
+public record QuestionRow(
+ string Id,
+ string Widget,
+ string Difficulty,
+ string Question,
+ Icons ResultIcon,
+ string Status,
+ string Time
+);
+
+/// Latest row in ivy_ask_test_runs (for Run Tests summary UI).
+public sealed record LastSavedRunSummary(
+ Guid Id,
+ string IvyVersion,
+ string Environment,
+ string DifficultyFilter,
+ string Concurrency,
+ int TotalQuestions,
+ int SuccessCount,
+ int NoAnswerCount,
+ int ErrorCount,
+ DateTime StartedAtUtc,
+ DateTime? CompletedAtUtc,
+ IReadOnlyList Rows);
+
+public sealed record LastSavedRunResultRow(
+ string Widget,
+ string Difficulty,
+ string QuestionPreview,
+ string Outcome,
+ int ResponseTimeMs);
diff --git a/project-demos/ivy-ask-statistics/Models/WidgetRow.cs b/project-demos/ivy-ask-statistics/Models/WidgetRow.cs
new file mode 100644
index 00000000..35a55c8f
--- /dev/null
+++ b/project-demos/ivy-ask-statistics/Models/WidgetRow.cs
@@ -0,0 +1,10 @@
+namespace IvyAskStatistics.Models;
+
+public record WidgetRow(
+ string Widget,
+ string Category,
+ int Easy,
+ int Medium,
+ int Hard,
+ string LastUpdated,
+ string Status = "");
diff --git a/project-demos/ivy-ask-statistics/Program.cs b/project-demos/ivy-ask-statistics/Program.cs
new file mode 100644
index 00000000..094ad4d4
--- /dev/null
+++ b/project-demos/ivy-ask-statistics/Program.cs
@@ -0,0 +1,27 @@
+using IvyAskStatistics.Apps;
+
+CultureInfo.DefaultThreadCurrentCulture = CultureInfo.DefaultThreadCurrentUICulture = new CultureInfo("en-US");
+
+var server = new Server();
+#if DEBUG
+server.UseHotReload();
+#endif
+server.AddAppsFromAssembly();
+server.AddConnectionsFromAssembly();
+
+var appShellSettings = new AppShellSettings()
+ .DefaultApp()
+ .UseTabs(preventDuplicates: true)
+ .UseFooterMenuItemsTransformer((items, navigator) =>
+ {
+ var list = items.ToList();
+ list.Add(MenuItem.Default("Generate All Questions").Icon(Icons.Sparkles).OnSelect(() =>
+ {
+ GenerateAllBridge.Request();
+ navigator.Navigate(typeof(QuestionsApp));
+ }));
+ return list;
+ });
+
+server.UseAppShell(appShellSettings);
+await server.RunAsync();
diff --git a/project-demos/ivy-ask-statistics/README.md b/project-demos/ivy-ask-statistics/README.md
new file mode 100644
index 00000000..09157fc6
--- /dev/null
+++ b/project-demos/ivy-ask-statistics/README.md
@@ -0,0 +1,125 @@
+# IVY Ask Statistics
+
+Automated quality testing for the IVY Ask API (`mcp.ivy.app`).
+
+Sends a curated set of questions to the API, measures response time, and reports which questions return answers and which don't. Designed to run before/after releases to catch regressions.
+
+## What it does
+
+- Calls `GET https://mcp.ivy.app/questions?question=` for every question in `questions/questions.json`
+- Classifies each result:
+ - **success** — HTTP 200 with a markdown answer
+ - **no_answer** — HTTP 404, API returned `NO_ANSWER_FOUND`
+ - **error** — anything else (network issue, bad request, etc.)
+- Measures response time per question (avg + P90)
+- Saves a full JSON result file to `results/`
+- Prints a summary to stdout
+- Writes a markdown table to `$GITHUB_STEP_SUMMARY` when running in GitHub Actions
+
+## Quick start
+
+```bash
+# Run all 35 questions against production
+./run-tests.sh
+
+# Filter by difficulty
+DIFFICULTY=hard ./run-tests.sh
+
+# Run against staging
+ENV=staging BASE_URL=https://mcp-staging.ivy.app ./run-tests.sh
+```
+
+**Requirements:** `jq`, `curl`, `python3` (all standard on macOS and Ubuntu).
+
+## Questions dataset
+
+`questions/questions.json` — 35 questions across 14 widgets, 3 difficulty levels each.
+
+| Difficulty | Description |
+|-----------|-------------|
+| `easy` | Basic widget usage — should always return an answer |
+| `medium` | Specific features and configuration |
+| `hard` | Advanced patterns, edge cases, compositions |
+
+To add or edit questions, just edit `questions/questions.json`. Each entry:
+
+```json
+{
+ "id": "button-easy-1",
+ "widget": "button",
+ "difficulty": "easy",
+ "question": "how to create a Button with an onClick handler in Ivy?"
+}
+```
+
+- `id` must be unique
+- `widget` is used for grouping in the summary
+- `difficulty` must be `easy`, `medium`, or `hard`
+- `question` should be a single, simple question (no compound questions)
+
+## Results format
+
+Each run writes a timestamped JSON file to `results/` (gitignored):
+
+```
+results/results-production-20260328-160000.json
+```
+
+Structure:
+
+```json
+{
+ "meta": {
+ "timestamp": "2026-03-28T16:00:00Z",
+ "environment": "production",
+ "total": 35,
+ "success": 30,
+ "no_answer": 5,
+ "errors": 0,
+ "successRatePct": 86,
+ "avgResponseMs": 1200,
+ "p90ResponseMs": 2400,
+ "byDifficulty": [...],
+ "byWidget": [...]
+ },
+ "results": [
+ {
+ "id": "button-easy-1",
+ "widget": "button",
+ "difficulty": "easy",
+ "question": "how to create a Button with an onClick handler in Ivy?",
+ "status": "success",
+ "responseTimeMs": 980,
+ "httpStatus": "200",
+ "timestamp": "2026-03-28T16:00:00Z",
+ "environment": "production"
+ }
+ ]
+}
+```
+
+## CI / GitHub Actions
+
+The workflow at `.github/workflows/ivy-ask-statistics.yml` supports:
+
+**Manual trigger** — go to Actions → IVY Ask Statistics → Run workflow:
+- Choose environment: `production` or `staging`
+- Optionally filter by difficulty
+
+**Scheduled** — runs every Monday at 08:00 UTC against production.
+
+Results are uploaded as a GitHub Actions artifact (90-day retention) and displayed as a table in the job summary.
+
+### Staging URL
+
+Set the `IVY_ASK_STAGING_URL` repository secret to override the default staging URL.
+
+## Environment variables
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `ENV` | `production` | Label written to results |
+| `BASE_URL` | `https://mcp.ivy.app` | API base URL |
+| `DIFFICULTY` | _(all)_ | Filter: `easy`, `medium`, or `hard` |
+| `QUESTIONS_FILE` | `questions/questions.json` | Path to questions dataset |
+| `RESULTS_DIR` | `results/` | Where to write result files |
diff --git a/project-demos/ivy-ask-statistics/Services/GenerateAllBridge.cs b/project-demos/ivy-ask-statistics/Services/GenerateAllBridge.cs
new file mode 100644
index 00000000..6114c177
--- /dev/null
+++ b/project-demos/ivy-ask-statistics/Services/GenerateAllBridge.cs
@@ -0,0 +1,31 @@
+namespace IvyAskStatistics.Apps;
+
+public static class GenerateAllBridge
+{
+ static volatile bool _pending;
+ static Action? _flushFromQuestionsView;
+
+ ///
+ /// Footer: set pending and flush immediately if is already mounted.
+ /// Otherwise Navigate runs a build that flushes via the registered handler.
+ /// (Same-tab Navigate is often a no-op with preventDuplicates, so this avoids a missing dialog.)
+ ///
+ public static void Request()
+ {
+ _pending = true;
+ _flushFromQuestionsView?.Invoke();
+ }
+
+ /// True while a footer request is pending and not yet consumed.
+ public static bool IsPending => _pending;
+
+ /// Latest flush from OnBuild; cleared on unmount.
+ public static void SetFlushHandler(Action? handler) => _flushFromQuestionsView = handler;
+
+ public static bool Consume()
+ {
+ if (!_pending) return false;
+ _pending = false;
+ return true;
+ }
+}
diff --git a/project-demos/ivy-ask-statistics/Services/IvyAskService.cs b/project-demos/ivy-ask-statistics/Services/IvyAskService.cs
new file mode 100644
index 00000000..2cfd0257
--- /dev/null
+++ b/project-demos/ivy-ask-statistics/Services/IvyAskService.cs
@@ -0,0 +1,240 @@
+using System.Diagnostics;
+using System.Text.Json;
+using System.Text.RegularExpressions;
+using IvyAskStatistics.Connections;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Configuration;
+
+namespace IvyAskStatistics.Services;
+
+public static class IvyAskService
+{
+ public const string DefaultMcpBaseUrl = "https://mcp.ivy.app";
+
+ ///
+ /// Sent as client= on every MCP HTTP request when no override is provided.
+ /// Use a stable slug: lowercase, ASCII, hyphens (e.g. ivy-internal, acme-corp-qa).
+ ///
+ public const string DefaultMcpClientId = "ivy-internal";
+
+ private static readonly HttpClient _http = new() { Timeout = TimeSpan.FromSeconds(120) };
+
+ /// Builds GET .../questions?question=...&client=... (client is always included).
+ public static string BuildQuestionsUrl(string baseUrl, string questionOrPrompt, string? client = null)
+ {
+ var root = baseUrl.TrimEnd('/');
+ var q = Uri.EscapeDataString(questionOrPrompt);
+ var id = NormalizeMcpClient(client);
+ return $"{root}/questions?question={q}&client={Uri.EscapeDataString(id)}";
+ }
+
+ /// Normalizes override or returns .
+ public static string NormalizeMcpClient(string? client)
+ {
+ var t = (client ?? "").Trim();
+ if (t.Length == 0) return DefaultMcpClientId;
+ if (t.Length > 120) return t[..120];
+ return t;
+ }
+
+ ///
+ /// MCP client query value from Mcp:ClientId (user secrets, env, appsettings).
+ /// If unset or blank, uses .
+ ///
+ public static string ResolveMcpClientId(IConfiguration configuration) =>
+ NormalizeMcpClient(configuration["Mcp:ClientId"]);
+
+ /// Appends client= to any MCP URL (e.g. /docs, widget doc path).
+ public static string WithMcpClientQuery(string absoluteOrRootRelativeUrl, string? client = null)
+ {
+ var id = NormalizeMcpClient(client);
+ var url = absoluteOrRootRelativeUrl.Trim();
+ var sep = url.Contains('?', StringComparison.Ordinal) ? "&" : "?";
+ return $"{url}{sep}client={Uri.EscapeDataString(id)}";
+ }
+
+ ///
+ /// Sends a question to the IVY Ask API and returns the result with timing.
+ /// GET {baseUrl}/questions?question={encoded}&client={client}
+ ///
+ /// Status codes:
+ /// 200 + body → "success" (answer found)
+ /// 404 → "no_answer" (NO_ANSWER_FOUND)
+ /// other → "error"
+ ///
+ public static async Task AskAsync(TestQuestion question, string baseUrl, string? client = null)
+ {
+ var url = BuildQuestionsUrl(baseUrl, question.Question, client);
+
+ var sw = Stopwatch.StartNew();
+ try
+ {
+ var response = await _http.GetAsync(url);
+ sw.Stop();
+
+ var body = await response.Content.ReadAsStringAsync();
+ var ms = (int)sw.ElapsedMilliseconds;
+ var httpStatus = (int)response.StatusCode;
+
+ var status = response.StatusCode switch
+ {
+ HttpStatusCode.OK when !string.IsNullOrWhiteSpace(body) => "success",
+ HttpStatusCode.NotFound => "no_answer",
+ _ => "error"
+ };
+
+ var answerText = status == "success" ? body : "";
+ return new QuestionRun(question, status, ms, httpStatus, answerText);
+ }
+ catch
+ {
+ sw.Stop();
+ return new QuestionRun(question, "error", (int)sw.ElapsedMilliseconds, 0);
+ }
+ }
+
+ ///
+ /// Calls Ivy Ask with a meta-prompt (e.g. “output JSON array of test questions”).
+ /// Same endpoint as normal Ask; the service returns generated text (often JSON).
+ ///
+ public static async Task FetchAnswerBodyAsync(
+ string baseUrl, string prompt, CancellationToken ct = default, string? client = null)
+ {
+ var url = BuildQuestionsUrl(baseUrl, prompt, client);
+ using var response = await _http.GetAsync(url, ct);
+ if (response.StatusCode != HttpStatusCode.OK) return null;
+ return await response.Content.ReadAsStringAsync(ct);
+ }
+
+ ///
+ /// Parses Ivy Ask response body into a list of question strings.
+ /// Handles raw JSON array, {"questions":[...]}, or fenced markdown code blocks.
+ ///
+ public static List ParseQuestionStringsFromBody(string body, int take = 10)
+ {
+ if (string.IsNullOrWhiteSpace(body)) return [];
+
+ var trimmed = body.Trim();
+
+ // Strip ```json ... ``` if present
+ var fence = Regex.Match(trimmed, @"^```(?:json)?\s*([\s\S]*?)\s*```", RegexOptions.IgnoreCase);
+ if (fence.Success)
+ trimmed = fence.Groups[1].Value.Trim();
+
+ try
+ {
+ using var doc = JsonDocument.Parse(trimmed);
+ var root = doc.RootElement;
+
+ if (root.ValueKind == JsonValueKind.Array)
+ return TakeStrings(root, take);
+
+ if (root.ValueKind == JsonValueKind.Object)
+ {
+ foreach (var name in new[] { "questions", "items", "data" })
+ {
+ if (root.TryGetProperty(name, out var arr) && arr.ValueKind == JsonValueKind.Array)
+ return TakeStrings(arr, take);
+ }
+ }
+ }
+ catch (JsonException)
+ {
+ // One question per non-empty line
+ return trimmed
+ .Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
+ .Select(l => l.TrimStart('-', ' ', '\t', '"').TrimEnd('"', ','))
+ .Where(l => l.Length > 5)
+ .Take(take)
+ .ToList();
+ }
+
+ return [];
+ }
+
+ private static List TakeStrings(JsonElement array, int take) =>
+ array.EnumerateArray()
+ .Select(e => e.ValueKind == JsonValueKind.String ? e.GetString() ?? "" : e.GetRawText())
+ .Where(s => !string.IsNullOrWhiteSpace(s))
+ .Take(take)
+ .ToList();
+
+ ///
+ /// Fetches the full list of Ivy widgets from the docs API.
+ /// Returns widgets only (filters out non-widget topics).
+ ///
+ public static async Task> GetWidgetsAsync(string? client = null)
+ {
+ var yaml = await _http.GetStringAsync(WithMcpClientQuery($"{DefaultMcpBaseUrl}/docs", client));
+ var widgets = new List();
+
+ var lines = yaml.Split('\n');
+ for (var i = 0; i < lines.Length - 1; i++)
+ {
+ var nameLine = lines[i].Trim();
+ if (!nameLine.StartsWith("- name: Ivy.Widgets.")) continue;
+
+ var fullName = nameLine["- name: ".Length..];
+ var linkLine = lines[i + 1].Trim();
+ var link = linkLine.StartsWith("link: ") ? linkLine["link: ".Length..] : "";
+
+ var parts = fullName.Split('.');
+ if (parts.Length < 4) continue;
+
+ var category = parts[2];
+ var name = parts[3];
+
+ widgets.Add(new IvyWidget(name, category, link.Trim()));
+ }
+
+ return widgets.OrderBy(w => w.Category).ThenBy(w => w.Name).ToList();
+ }
+
+ ///
+ /// Widgets from /docs merged with distinct widget names stored in the database.
+ /// If the docs API fails or returns nothing, rows still appear from DB (fixes empty table when offline).
+ ///
+ public static async Task> GetMergedWidgetCatalogAsync(
+ AppDbContextFactory factory,
+ CancellationToken ct = default)
+ {
+ List api = [];
+ try
+ {
+ api = await GetWidgetsAsync();
+ }
+ catch
+ {
+ // e.g. network blocked in browser / server
+ }
+
+ await using var ctx = factory.CreateDbContext();
+ var fromDb = await ctx.Questions
+ .AsNoTracking()
+ .Where(q => q.Widget != "")
+ .GroupBy(q => q.Widget)
+ .Select(g => new { Widget = g.Key, Category = g.Select(x => x.Category).FirstOrDefault() })
+ .ToListAsync(ct);
+
+ var byName = api.ToDictionary(w => w.Name, w => w);
+ foreach (var row in fromDb)
+ {
+ if (!byName.ContainsKey(row.Widget))
+ byName[row.Widget] = new IvyWidget(row.Widget, row.Category ?? "", "");
+ }
+
+ return byName.Values
+ .OrderBy(w => w.Category)
+ .ThenBy(w => w.Name)
+ .ToList();
+ }
+
+ ///
+ /// Fetches the Markdown documentation for a specific widget.
+ ///
+ public static async Task GetWidgetDocsAsync(string docLink, string? client = null)
+ {
+ var path = docLink.TrimStart('/');
+ return await _http.GetStringAsync(WithMcpClientQuery($"{DefaultMcpBaseUrl}/{path}", client));
+ }
+}
diff --git a/project-demos/ivy-ask-statistics/Services/QuestionGeneratorService.cs b/project-demos/ivy-ask-statistics/Services/QuestionGeneratorService.cs
new file mode 100644
index 00000000..ec2b0ae2
--- /dev/null
+++ b/project-demos/ivy-ask-statistics/Services/QuestionGeneratorService.cs
@@ -0,0 +1,152 @@
+using IvyAskStatistics.Connections;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Configuration;
+using OpenAI;
+using OpenAI.Chat;
+using System.ClientModel;
+
+namespace IvyAskStatistics.Services;
+
+///
+/// Generates test questions by fetching the full widget Markdown documentation
+/// from mcp.ivy.app and passing it to OpenAI as context.
+/// Uses OpenAI:ApiKey, OpenAI:BaseUrl, and OpenAI:ChatModel from configuration / user secrets.
+///
+public static class QuestionGeneratorService
+{
+ public const string ApiKeyConfigKey = "OpenAI:ApiKey";
+ public const string BaseUrlConfigKey = "OpenAI:BaseUrl";
+ public const string ChatModelConfigKey = "OpenAI:ChatModel";
+
+ /// Returns null if OpenAI settings are present; otherwise a user-facing message.
+ public static string? GetOpenAiConfigurationError(IConfiguration configuration)
+ {
+ if (string.IsNullOrWhiteSpace(configuration[ApiKeyConfigKey]))
+ return $"{ApiKeyConfigKey} is not set. Run: dotnet user-secrets set \"{ApiKeyConfigKey}\" \"\"";
+ if (string.IsNullOrWhiteSpace(configuration[BaseUrlConfigKey]))
+ return $"{BaseUrlConfigKey} is not set. Run: dotnet user-secrets set \"{BaseUrlConfigKey}\" \"\"";
+ if (string.IsNullOrWhiteSpace(configuration[ChatModelConfigKey]?.Trim()))
+ return $"{ChatModelConfigKey} is not set. Run: dotnet user-secrets set \"{ChatModelConfigKey}\" \"\"";
+ return null;
+ }
+
+ ///
+ /// Generates 10 questions per difficulty (easy / medium / hard) for a widget
+ /// and saves them to the database, replacing any previously generated ones.
+ ///
+ public static async Task GenerateAndSaveAsync(
+ IvyWidget widget,
+ AppDbContextFactory factory,
+ string openAiApiKey,
+ string openAiBaseUrl,
+ IConfiguration configuration,
+ IProgress? progress = null,
+ CancellationToken ct = default)
+ {
+ var cfgErr = GetOpenAiConfigurationError(configuration);
+ if (cfgErr != null)
+ throw new InvalidOperationException(cfgErr);
+
+ progress?.Report($"Fetching docs for {widget.Name}…");
+
+ var markdown = await FetchDocsMarkdownAsync(widget, ct);
+
+ var model = configuration[ChatModelConfigKey]!.Trim();
+ var chatClient = BuildChatClient(openAiApiKey, openAiBaseUrl, model);
+
+ foreach (var difficulty in new[] { "easy", "medium", "hard" })
+ {
+ progress?.Report($"OpenAI: generating {difficulty} questions for {widget.Name}…");
+
+ var messages = BuildMessages(widget, difficulty, markdown);
+ var response = await chatClient.CompleteChatAsync(messages, cancellationToken: ct);
+ var body = response.Value.Content[0].Text ?? "";
+
+ var questions = IvyAskService.ParseQuestionStringsFromBody(body, 10);
+ if (questions.Count == 0)
+ throw new InvalidOperationException(
+ $"Could not parse {difficulty} questions from OpenAI response. Body: {body[..Math.Min(200, body.Length)]}");
+
+ await using var ctx = factory.CreateDbContext();
+
+ var existing = await ctx.Questions
+ .Where(q => q.Widget == widget.Name && q.Difficulty == difficulty && q.Source == "openai_docs")
+ .ToListAsync(ct);
+ ctx.Questions.RemoveRange(existing);
+
+ ctx.Questions.AddRange(questions.Select(q => new QuestionEntity
+ {
+ Widget = widget.Name,
+ Category = widget.Category,
+ Difficulty = difficulty,
+ QuestionText = q,
+ Source = "openai_docs",
+ CreatedAt = DateTime.UtcNow
+ }));
+
+ await ctx.SaveChangesAsync(ct);
+ }
+ }
+
+ private static async Task FetchDocsMarkdownAsync(IvyWidget widget, CancellationToken ct)
+ {
+ if (string.IsNullOrWhiteSpace(widget.DocLink))
+ return $"No documentation link available for widget \"{widget.Name}\".";
+
+ try
+ {
+ return await IvyAskService.GetWidgetDocsAsync(widget.DocLink);
+ }
+ catch (Exception ex)
+ {
+ throw new InvalidOperationException(
+ $"Failed to fetch docs for \"{widget.Name}\" (link: {widget.DocLink}): {ex.Message}", ex);
+ }
+ }
+
+ private static ChatClient BuildChatClient(string apiKey, string baseUrl, string model)
+ {
+ var options = new OpenAIClientOptions
+ {
+ Endpoint = new Uri(baseUrl.TrimEnd('/'))
+ };
+ var client = new OpenAIClient(new ApiKeyCredential(apiKey), options);
+ return client.GetChatClient(model);
+ }
+
+ private static List BuildMessages(IvyWidget widget, string difficulty, string markdown)
+ {
+ var difficultyHint = difficulty switch
+ {
+ "easy" => "basic usage, creating the widget, simple properties",
+ "medium" => "events, styling, configuration, common patterns",
+ _ => "advanced composition, edge cases, integration with other Ivy widgets"
+ };
+
+ var system = new SystemChatMessage("""
+ You are generating automated test questions for the Ivy UI framework documentation QA system.
+ You will receive the full Markdown documentation for a specific Ivy widget.
+ Base your questions ONLY on what is documented in the provided content.
+ Do not invent APIs, properties, or behaviors not mentioned in the documentation.
+ """);
+
+ var user = new UserChatMessage($"""
+ Widget name: {widget.Name}
+ Widget category: {widget.Category}
+ Difficulty: {difficulty} — {difficultyHint}
+
+ --- DOCUMENTATION ---
+ {markdown}
+ --- END DOCUMENTATION ---
+
+ Generate exactly 10 distinct {difficulty}-level questions that a C# developer might ask about the "{widget.Name}" widget.
+ Each question must be a single sentence, under 25 words, and grounded in the documentation above.
+ Do not combine multiple unrelated topics in one question.
+
+ Respond with ONLY a JSON array of 10 strings (no markdown fences, no keys, no commentary).
+ Example shape: ["question1?", "question2?", ...]
+ """);
+
+ return [system, user];
+ }
+}