From df2f68829c63ac5bca26d4cc52ca164bacba8e44 Mon Sep 17 00:00:00 2001 From: Ben Lubas Date: Thu, 22 Jan 2026 14:15:47 -0500 Subject: [PATCH 1/4] feat: add q comment string --- src/core/jupyter/jupyter.ts | 1 + src/core/lib/partition-cell-options.ts | 1 + src/resources/filters/modules/constants.lua | 1 + src/resources/jupyter/notebook.py | 1 + src/resources/rmd/hooks.R | 1 + 5 files changed, 5 insertions(+) diff --git a/src/core/jupyter/jupyter.ts b/src/core/jupyter/jupyter.ts index 765774e4d23..8ec7f336c07 100644 --- a/src/core/jupyter/jupyter.ts +++ b/src/core/jupyter/jupyter.ts @@ -1249,6 +1249,7 @@ const kLangCommentChars: Record = { mermaid: "%%", apl: "⍝", ocaml: ["(*", "*)"], + q: "/", rust: "//", }; diff --git a/src/core/lib/partition-cell-options.ts b/src/core/lib/partition-cell-options.ts index c066fe57a03..8d056f5a283 100644 --- a/src/core/lib/partition-cell-options.ts +++ b/src/core/lib/partition-cell-options.ts @@ -353,6 +353,7 @@ export const kLangCommentChars: Record = { ojs: "//", apl: "⍝", ocaml: ["(*", "*)"], + q: "/", rust: "//", }; diff --git a/src/resources/filters/modules/constants.lua b/src/resources/filters/modules/constants.lua index 42a59d77b4c..e22a6774417 100644 --- a/src/resources/filters/modules/constants.lua +++ b/src/resources/filters/modules/constants.lua @@ -116,6 +116,7 @@ local kLangCommentChars = { powershell = {"#"}, psql = {"--"}, python = {"#"}, + q = {"/"}, r = {"#"}, ruby = {"#"}, rust = {"//"}, diff --git a/src/resources/jupyter/notebook.py b/src/resources/jupyter/notebook.py index 2d4ca301525..4ac5383e004 100644 --- a/src/resources/jupyter/notebook.py +++ b/src/resources/jupyter/notebook.py @@ -875,6 +875,7 @@ def nb_language_comment_chars(lang): haskell="--", dot="//", apl="⍝", + q = "/", ocaml=["(*", "*)"], ) if lang in langs: diff --git a/src/resources/rmd/hooks.R b/src/resources/rmd/hooks.R index 7e3ca19dbd9..11ab12e33a9 100644 --- a/src/resources/rmd/hooks.R +++ b/src/resources/rmd/hooks.R @@ -1046,6 +1046,7 @@ engine_comment_chars <- function(engine) { dot = "//", apl = "\u235D", ocaml = c("(*", "*)"), + q = "/", rust = "//" ) comment_chars[[engine]] %||% "#" From 2d5587e74c26da82c5ff1fb12cfa701a7afa19f4 Mon Sep 17 00:00:00 2001 From: Ben Lubas Date: Thu, 22 Jan 2026 14:15:47 -0500 Subject: [PATCH 2/4] feat: support q percent format Additionally, make it possible for quarto to read percent style notebooks from languages that don't use `#` as the comment char --- src/core/jupyter/percent.ts | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/src/core/jupyter/percent.ts b/src/core/jupyter/percent.ts index b41416949cd..586f61ae610 100644 --- a/src/core/jupyter/percent.ts +++ b/src/core/jupyter/percent.ts @@ -12,19 +12,32 @@ import { kCellRawMimeType } from "../../config/constants.ts"; import { mdFormatOutput, mdRawOutput } from "./jupyter.ts"; import { lines, trimEmptyLines } from "../lib/text.ts"; import { asYamlText } from "./jupyter-fixups.ts"; +import { kLangCommentChars } from "../lib/partition-cell-options.ts"; export const kJupyterPercentScriptExtensions = [ ".py", ".jl", ".r", + ".q", ]; +export const kLanguageExtensions: Record = { + ".jl": "julia", + ".r": "r", + ".py": "python", + ".q": "q", +}; + + export function isJupyterPercentScript(file: string, extensions?: string[]) { const ext = extname(file).toLowerCase(); const availableExtensions = extensions ?? kJupyterPercentScriptExtensions; if (availableExtensions.includes(ext)) { const text = Deno.readTextFileSync(file); - return !!text.match(/^\s*#\s*%%+\s+\[markdown|raw\]/); + const cms = kLangCommentChars[kLanguageExtensions[ext]]; + const pat = new RegExp(`^\\s*${cms}\\s*%%+\\s+\\[markdown|raw\\]`); + return !!text.match(pat); + } else { return false; } @@ -33,13 +46,17 @@ export function isJupyterPercentScript(file: string, extensions?: string[]) { export function markdownFromJupyterPercentScript(file: string) { // determine language/kernel const ext = extname(file).toLowerCase(); - const language = ext === ".jl" ? "julia" : ext === ".r" ? "r" : "python"; + const language = kLanguageExtensions[ext]; + const cms = kLangCommentChars[language]; + if (!language) { + throw new Error(`Could not determine language from file extension ${ext}`); + } // break into cells const cells: PercentCell[] = []; const activeCell = () => cells[cells.length - 1]; for (const line of lines(Deno.readTextFileSync(file).trim())) { - const header = percentCellHeader(line); + const header = percentCellHeader(line, language); if (header) { cells.push({ header, lines: [] }); } else { @@ -50,7 +67,7 @@ export function markdownFromJupyterPercentScript(file: string) { // resolve markdown and raw cells const isTripleQuote = (line: string) => !!line.match(/^"{3,}\s*$/); const asCell = (lines: string[]) => lines.join("\n") + "\n\n"; - const stripPrefix = (line: string) => line.replace(/^#\s?/, ""); + const stripPrefix = (line: string) => line.replace(new RegExp(`^${cms}\\s?`), ""); const cellContent = (cellLines: string[]) => { if ( cellLines.length > 2 && isTripleQuote(cellLines[0]) && @@ -68,7 +85,7 @@ export function markdownFromJupyterPercentScript(file: string) { if (cell.header.type === "code") { if (cell.header.metadata) { const yamlText = asYamlText(cell.header.metadata); - cellLines.unshift(...lines(yamlText).map((line) => `#| ${line}`)); + cellLines.unshift(...lines(yamlText).map((line) => `${cms}| ${line}`)); } markdown += asCell(["```{" + language + "}", ...cellLines, "```"]); } else if (cell.header.type === "markdown") { @@ -99,9 +116,10 @@ interface PercentCellHeader { metadata?: Metadata; } -function percentCellHeader(line: string): PercentCellHeader | undefined { +function percentCellHeader(line: string, language: string): PercentCellHeader | undefined { + const cms = kLangCommentChars[language]; const match = line.match( - /^\s*#\s*%%+\s*(?:\[(markdown|raw)\])?\s*(.*)?$/, + new RegExp(`^\\s*${cms}\\s*%%+\\s*(?:\\[(markdown|raw)\\])?\\s*(.*)?$`), ); if (match) { const type = match[1] || "code"; From 6ddfcbfb3f3cbec79162b563d7d2b9ec63b43aa5 Mon Sep 17 00:00:00 2001 From: Ben Lubas Date: Thu, 22 Jan 2026 18:04:37 -0500 Subject: [PATCH 3/4] fix: unrelated logging error --- src/resources/filters/common/log.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resources/filters/common/log.lua b/src/resources/filters/common/log.lua index 67a736bbe93..1fad6971a2d 100644 --- a/src/resources/filters/common/log.lua +++ b/src/resources/filters/common/log.lua @@ -25,7 +25,7 @@ function warn(message, offset) end function error(message, offset) - io.stderr:write(lunacolors.red("ERROR (" .. caller_info(offset) .. ") " .. message .. "\n")) + io.stderr:write(lunacolors.red(("ERROR (%s) %s\n"):format(caller_info(offset), message))) end function fatal(message, offset) From 68d763c5dfa8605ecc92a001ad0103df819c9096 Mon Sep 17 00:00:00 2001 From: Carlos Scheidegger Date: Tue, 27 Jan 2026 13:44:33 -0600 Subject: [PATCH 4/4] Add unit tests for percent format and fix regex bug - Add comprehensive unit tests for percent format notebook parsing - Test detection and conversion for Python, Julia, R, and q languages - Fix isJupyterPercentScript regex: properly group alternation (markdown|raw) and use multiline mode so ^ matches start of any line - Add changelog entries for q/kdb+ support and regex fix Co-Authored-By: Claude Opus 4.5 --- news/changelog-1.9.md | 2 + src/core/jupyter/percent.ts | 5 +- .../docs/percent-format/false-positive-raw.py | 6 + tests/docs/percent-format/not-percent.py | 7 + tests/docs/percent-format/not-percent.q | 5 + tests/docs/percent-format/test-julia.jl | 11 + .../percent-format/test-markdown-not-first.py | 13 ++ tests/docs/percent-format/test-python.py | 11 + tests/docs/percent-format/test-q.q | 11 + tests/docs/percent-format/test-r.r | 11 + tests/docs/percent-format/test-raw-cell.py | 5 + tests/unit/percent-format.test.ts | 190 ++++++++++++++++++ 12 files changed, 275 insertions(+), 2 deletions(-) create mode 100644 tests/docs/percent-format/false-positive-raw.py create mode 100644 tests/docs/percent-format/not-percent.py create mode 100644 tests/docs/percent-format/not-percent.q create mode 100644 tests/docs/percent-format/test-julia.jl create mode 100644 tests/docs/percent-format/test-markdown-not-first.py create mode 100644 tests/docs/percent-format/test-python.py create mode 100644 tests/docs/percent-format/test-q.q create mode 100644 tests/docs/percent-format/test-r.r create mode 100644 tests/docs/percent-format/test-raw-cell.py create mode 100644 tests/unit/percent-format.test.ts diff --git a/news/changelog-1.9.md b/news/changelog-1.9.md index 5996e470f2e..90f858853d9 100644 --- a/news/changelog-1.9.md +++ b/news/changelog-1.9.md @@ -124,6 +124,8 @@ All changes included in 1.9: ### `jupyter` - ([#13748](https://github.com/quarto-dev/quarto-cli/pull/13748)): Fix stdin encoding to UTF-8 on Windows to correctly handle JSON in documents containing non-ASCII characters. +- ([#13936](https://github.com/quarto-dev/quarto-cli/pull/13936)): Add support for q/kdb+ programming language in percent format notebooks and code cell options. (author: @benlubas) +- ([#13936](https://github.com/quarto-dev/quarto-cli/pull/13936)): Fix `isJupyterPercentScript` regex to correctly detect percent scripts with `[raw]` cells and cells not at the start of the file. The regex now properly groups the alternation `(markdown|raw)` and uses multiline mode. ## Other fixes and improvements diff --git a/src/core/jupyter/percent.ts b/src/core/jupyter/percent.ts index 586f61ae610..44357b38b81 100644 --- a/src/core/jupyter/percent.ts +++ b/src/core/jupyter/percent.ts @@ -35,9 +35,10 @@ export function isJupyterPercentScript(file: string, extensions?: string[]) { if (availableExtensions.includes(ext)) { const text = Deno.readTextFileSync(file); const cms = kLangCommentChars[kLanguageExtensions[ext]]; - const pat = new RegExp(`^\\s*${cms}\\s*%%+\\s+\\[markdown|raw\\]`); + // Use multiline mode (m) so ^ matches start of any line, not just start of file. + // Group the alternation properly: (markdown|raw) not markdown|raw + const pat = new RegExp(`^\\s*${cms}\\s*%%+\\s+\\[(markdown|raw)\\]`, "m"); return !!text.match(pat); - } else { return false; } diff --git a/tests/docs/percent-format/false-positive-raw.py b/tests/docs/percent-format/false-positive-raw.py new file mode 100644 index 00000000000..84139b38e42 --- /dev/null +++ b/tests/docs/percent-format/false-positive-raw.py @@ -0,0 +1,6 @@ +# This file contains the text "raw]" but is NOT a percent script +# It should test that the regex doesn't match "raw]" anywhere in the file +# For example: handle raw] data or [raw] text in comments + +def process(): + return "raw]" diff --git a/tests/docs/percent-format/not-percent.py b/tests/docs/percent-format/not-percent.py new file mode 100644 index 00000000000..95f0b7ed880 --- /dev/null +++ b/tests/docs/percent-format/not-percent.py @@ -0,0 +1,7 @@ +# This is a regular Python file, not a percent script +# It has no special cell markers + +def hello(): + print("Hello") + +hello() diff --git a/tests/docs/percent-format/not-percent.q b/tests/docs/percent-format/not-percent.q new file mode 100644 index 00000000000..d659e5ca15c --- /dev/null +++ b/tests/docs/percent-format/not-percent.q @@ -0,0 +1,5 @@ +/ This is a regular q file, not a percent script +/ It has no special cell markers + +hello: {[] "Hello"} +hello[] diff --git a/tests/docs/percent-format/test-julia.jl b/tests/docs/percent-format/test-julia.jl new file mode 100644 index 00000000000..88a09a8a0e2 --- /dev/null +++ b/tests/docs/percent-format/test-julia.jl @@ -0,0 +1,11 @@ +# %% [markdown] +# This is a markdown cell + +# %% +println("Hello from Julia") + +# %% [markdown] +# Another markdown cell + +# %% +x = 1 + 1 diff --git a/tests/docs/percent-format/test-markdown-not-first.py b/tests/docs/percent-format/test-markdown-not-first.py new file mode 100644 index 00000000000..66ba1a5f650 --- /dev/null +++ b/tests/docs/percent-format/test-markdown-not-first.py @@ -0,0 +1,13 @@ +# --- +# jupyter: +# kernelspec: +# display_name: Python 3 +# language: python +# name: python3 +# --- + +# %% +print("Code first") + +# %% [markdown] +# This markdown cell is not at the start of the file diff --git a/tests/docs/percent-format/test-python.py b/tests/docs/percent-format/test-python.py new file mode 100644 index 00000000000..ea1bcad10a9 --- /dev/null +++ b/tests/docs/percent-format/test-python.py @@ -0,0 +1,11 @@ +# %% [markdown] +# This is a markdown cell + +# %% +print("Hello from Python") + +# %% [markdown] +# Another markdown cell + +# %% +x = 1 + 1 diff --git a/tests/docs/percent-format/test-q.q b/tests/docs/percent-format/test-q.q new file mode 100644 index 00000000000..93514e4aef7 --- /dev/null +++ b/tests/docs/percent-format/test-q.q @@ -0,0 +1,11 @@ +/ %% [markdown] +/ This is a markdown cell + +/ %% +1+1 + +/ %% [markdown] +/ Another markdown cell + +/ %% +x: 42 diff --git a/tests/docs/percent-format/test-r.r b/tests/docs/percent-format/test-r.r new file mode 100644 index 00000000000..620cf779174 --- /dev/null +++ b/tests/docs/percent-format/test-r.r @@ -0,0 +1,11 @@ +# %% [markdown] +# This is a markdown cell + +# %% +print("Hello from R") + +# %% [markdown] +# Another markdown cell + +# %% +x <- 1 + 1 diff --git a/tests/docs/percent-format/test-raw-cell.py b/tests/docs/percent-format/test-raw-cell.py new file mode 100644 index 00000000000..26780218a40 --- /dev/null +++ b/tests/docs/percent-format/test-raw-cell.py @@ -0,0 +1,5 @@ +# %% [raw] +# This is a raw cell + +# %% +print("Hello") diff --git a/tests/unit/percent-format.test.ts b/tests/unit/percent-format.test.ts new file mode 100644 index 00000000000..48aaecb762f --- /dev/null +++ b/tests/unit/percent-format.test.ts @@ -0,0 +1,190 @@ +/* + * percent-format.test.ts + * + * Tests for the Jupyter percent format script parsing + * + * Copyright (C) 2020-2024 Posit Software, PBC + */ +import { assertEquals, assert } from "testing/asserts"; +import { unitTest } from "../test.ts"; +import { docs } from "../utils.ts"; +import { + isJupyterPercentScript, + markdownFromJupyterPercentScript, + kJupyterPercentScriptExtensions, + kLanguageExtensions, +} from "../../src/core/jupyter/percent.ts"; + +// Test that the language extensions mapping is correct +unitTest("percent-format - kLanguageExtensions mapping", async () => { + assertEquals(kLanguageExtensions[".py"], "python"); + assertEquals(kLanguageExtensions[".jl"], "julia"); + assertEquals(kLanguageExtensions[".r"], "r"); + assertEquals(kLanguageExtensions[".q"], "q"); +}); + +// Test that all extensions in kJupyterPercentScriptExtensions have a language mapping +unitTest("percent-format - all extensions have language mappings", async () => { + for (const ext of kJupyterPercentScriptExtensions) { + assert( + kLanguageExtensions[ext] !== undefined, + `Extension ${ext} has no language mapping`, + ); + } +}); + +// Test isJupyterPercentScript for Python +unitTest("percent-format - isJupyterPercentScript detects Python percent script", async () => { + const result = isJupyterPercentScript(docs("percent-format/test-python.py")); + assertEquals(result, true, "Should detect Python percent script"); +}); + +// Test isJupyterPercentScript for q/kdb +unitTest("percent-format - isJupyterPercentScript detects q percent script", async () => { + const result = isJupyterPercentScript(docs("percent-format/test-q.q")); + assertEquals(result, true, "Should detect q percent script"); +}); + +// Test isJupyterPercentScript for Julia +unitTest("percent-format - isJupyterPercentScript detects Julia percent script", async () => { + const result = isJupyterPercentScript(docs("percent-format/test-julia.jl")); + assertEquals(result, true, "Should detect Julia percent script"); +}); + +// Test isJupyterPercentScript for R +unitTest("percent-format - isJupyterPercentScript detects R percent script", async () => { + const result = isJupyterPercentScript(docs("percent-format/test-r.r")); + assertEquals(result, true, "Should detect R percent script"); +}); + +// Test that non-percent scripts are not detected +unitTest("percent-format - isJupyterPercentScript rejects non-percent Python", async () => { + const result = isJupyterPercentScript(docs("percent-format/not-percent.py")); + assertEquals(result, false, "Should not detect non-percent Python file"); +}); + +unitTest("percent-format - isJupyterPercentScript rejects non-percent q", async () => { + const result = isJupyterPercentScript(docs("percent-format/not-percent.q")); + assertEquals(result, false, "Should not detect non-percent q file"); +}); + +// Test isJupyterPercentScript rejects unsupported extensions +unitTest("percent-format - isJupyterPercentScript rejects unsupported extensions", async () => { + // Create a temp file with unsupported extension - use a non-existent path + // The function checks extension first, so it won't try to read the file + const result = isJupyterPercentScript("/fake/path/file.txt"); + assertEquals(result, false, "Should reject unsupported extension"); +}); + +// Test markdownFromJupyterPercentScript for Python +unitTest("percent-format - markdownFromJupyterPercentScript converts Python", async () => { + const markdown = markdownFromJupyterPercentScript( + docs("percent-format/test-python.py"), + ); + + // Check that it contains python code blocks + assert(markdown.includes("```{python}"), "Should contain python code block"); + assert(markdown.includes("```"), "Should contain closing code fence"); + + // Check that markdown content is extracted + assert( + markdown.includes("This is a markdown cell"), + "Should contain markdown content", + ); + + // Check that code content is present + assert( + markdown.includes('print("Hello from Python")'), + "Should contain Python code", + ); +}); + +// Test markdownFromJupyterPercentScript for q/kdb +unitTest("percent-format - markdownFromJupyterPercentScript converts q", async () => { + const markdown = markdownFromJupyterPercentScript( + docs("percent-format/test-q.q"), + ); + + // Check that it contains q code blocks + assert(markdown.includes("```{q}"), "Should contain q code block"); + assert(markdown.includes("```"), "Should contain closing code fence"); + + // Check that markdown content is extracted (without the / prefix) + assert( + markdown.includes("This is a markdown cell"), + "Should contain markdown content", + ); + + // Check that code content is present + assert(markdown.includes("1+1"), "Should contain q code"); + assert(markdown.includes("x: 42"), "Should contain q code"); +}); + +// Test markdownFromJupyterPercentScript for Julia +unitTest("percent-format - markdownFromJupyterPercentScript converts Julia", async () => { + const markdown = markdownFromJupyterPercentScript( + docs("percent-format/test-julia.jl"), + ); + + // Check that it contains julia code blocks + assert(markdown.includes("```{julia}"), "Should contain julia code block"); + + // Check that markdown content is extracted + assert( + markdown.includes("This is a markdown cell"), + "Should contain markdown content", + ); + + // Check that code content is present + assert( + markdown.includes('println("Hello from Julia")'), + "Should contain Julia code", + ); +}); + +// Test markdownFromJupyterPercentScript for R +unitTest("percent-format - markdownFromJupyterPercentScript converts R", async () => { + const markdown = markdownFromJupyterPercentScript( + docs("percent-format/test-r.r"), + ); + + // Check that it contains r code blocks + assert(markdown.includes("```{r}"), "Should contain r code block"); + + // Check that markdown content is extracted + assert( + markdown.includes("This is a markdown cell"), + "Should contain markdown content", + ); + + // Check that code content is present + assert( + markdown.includes('print("Hello from R")'), + "Should contain R code", + ); +}); + +// ============================================================================= +// Regression tests for regex bug fix +// The original regex had incorrect alternation: \[markdown|raw\] which matches +// "[markdown" OR "raw]" instead of "[markdown]" OR "[raw]" +// ============================================================================= + +// Test that [raw] cells are detected (tests correct alternation grouping) +unitTest("percent-format - isJupyterPercentScript detects [raw] cells", async () => { + const result = isJupyterPercentScript(docs("percent-format/test-raw-cell.py")); + assertEquals(result, true, "Should detect percent script with [raw] cell"); +}); + +// Test that markdown cells not at start of file are detected (tests multiline matching) +unitTest("percent-format - isJupyterPercentScript detects markdown cell not at file start", async () => { + const result = isJupyterPercentScript(docs("percent-format/test-markdown-not-first.py")); + assertEquals(result, true, "Should detect percent script with markdown cell not at start"); +}); + +// Test that files containing "raw]" text are not falsely detected +// This was a bug where the regex would match "raw]" anywhere in the file +unitTest("percent-format - isJupyterPercentScript rejects file with raw] text", async () => { + const result = isJupyterPercentScript(docs("percent-format/false-positive-raw.py")); + assertEquals(result, false, "Should not detect file that merely contains 'raw]' text"); +});