feat: support for q/kdb programming language

news/changelog-1.9.md

@@ -124,6 +124,8 @@ All changes included in 1.9:
 ### `jupyter`
 
 - ([#13748](https://github.com/quarto-dev/quarto-cli/pull/13748)): Fix stdin encoding to UTF-8 on Windows to correctly handle JSON in documents containing non-ASCII characters.
+- ([#13936](https://github.com/quarto-dev/quarto-cli/pull/13936)): Add support for q/kdb+ programming language in percent format notebooks and code cell options. (author: @benlubas)
+- ([#13936](https://github.com/quarto-dev/quarto-cli/pull/13936)): Fix `isJupyterPercentScript` regex to correctly detect percent scripts with `[raw]` cells and cells not at the start of the file. The regex now properly groups the alternation `(markdown|raw)` and uses multiline mode.
 
 ## Other fixes and improvements
 

src/core/jupyter/jupyter.ts

@@ -1249,6 +1249,7 @@ const kLangCommentChars: Record<string, string | string[]> = {
   mermaid: "%%",
   apl: "⍝",
   ocaml: ["(*", "*)"],
+  q: "/",
   rust: "//",
 };
 

src/core/jupyter/percent.ts

@@ -12,19 +12,33 @@ import { kCellRawMimeType } from "../../config/constants.ts";
 import { mdFormatOutput, mdRawOutput } from "./jupyter.ts";
 import { lines, trimEmptyLines } from "../lib/text.ts";
 import { asYamlText } from "./jupyter-fixups.ts";
+import { kLangCommentChars } from "../lib/partition-cell-options.ts";
 
 export const kJupyterPercentScriptExtensions = [
   ".py",
   ".jl",
   ".r",
+  ".q",
 ];
 
+export const kLanguageExtensions: Record<string, string> = {
+  ".jl": "julia",
+  ".r": "r",
+  ".py": "python",
+  ".q": "q",
+};
+
+
 export function isJupyterPercentScript(file: string, extensions?: string[]) {
   const ext = extname(file).toLowerCase();
   const availableExtensions = extensions ?? kJupyterPercentScriptExtensions;
   if (availableExtensions.includes(ext)) {
     const text = Deno.readTextFileSync(file);
-    return !!text.match(/^\s*#\s*%%+\s+\[markdown|raw\]/);
+    const cms = kLangCommentChars[kLanguageExtensions[ext]];
+    // Use multiline mode (m) so ^ matches start of any line, not just start of file.
+    // Group the alternation properly: (markdown|raw) not markdown|raw
+    const pat = new RegExp(`^\\s*${cms}\\s*%%+\\s+\\[(markdown|raw)\\]`, "m");
+    return !!text.match(pat);
   } else {
     return false;
   }
@@ -33,13 +47,17 @@ export function isJupyterPercentScript(file: string, extensions?: string[]) {
 export function markdownFromJupyterPercentScript(file: string) {
   // determine language/kernel
   const ext = extname(file).toLowerCase();
-  const language = ext === ".jl" ? "julia" : ext === ".r" ? "r" : "python";
+  const language = kLanguageExtensions[ext];
+  const cms =  kLangCommentChars[language];
+  if (!language) {
+    throw new Error(`Could not determine language from file extension ${ext}`);
+  }
 
   // break into cells
   const cells: PercentCell[] = [];
   const activeCell = () => cells[cells.length - 1];
   for (const line of lines(Deno.readTextFileSync(file).trim())) {
-    const header = percentCellHeader(line);
+    const header = percentCellHeader(line, language);
     if (header) {
       cells.push({ header, lines: [] });
     } else {
@@ -50,7 +68,7 @@ export function markdownFromJupyterPercentScript(file: string) {
   // resolve markdown and raw cells
   const isTripleQuote = (line: string) => !!line.match(/^"{3,}\s*$/);
   const asCell = (lines: string[]) => lines.join("\n") + "\n\n";
-  const stripPrefix = (line: string) => line.replace(/^#\s?/, "");
+  const stripPrefix = (line: string) => line.replace(new RegExp(`^${cms}\\s?`), "");
   const cellContent = (cellLines: string[]) => {
     if (
       cellLines.length > 2 && isTripleQuote(cellLines[0]) &&
@@ -68,7 +86,7 @@ export function markdownFromJupyterPercentScript(file: string) {
     if (cell.header.type === "code") {
       if (cell.header.metadata) {
         const yamlText = asYamlText(cell.header.metadata);
-        cellLines.unshift(...lines(yamlText).map((line) => `#| ${line}`));
+        cellLines.unshift(...lines(yamlText).map((line) => `${cms}| ${line}`));
       }
       markdown += asCell(["```{" + language + "}", ...cellLines, "```"]);
     } else if (cell.header.type === "markdown") {
@@ -99,9 +117,10 @@ interface PercentCellHeader {
   metadata?: Metadata;
 }
 
-function percentCellHeader(line: string): PercentCellHeader | undefined {
+function percentCellHeader(line: string, language: string): PercentCellHeader | undefined {
+  const cms = kLangCommentChars[language];
   const match = line.match(
-    /^\s*#\s*%%+\s*(?:\[(markdown|raw)\])?\s*(.*)?$/,
+    new RegExp(`^\\s*${cms}\\s*%%+\\s*(?:\\[(markdown|raw)\\])?\\s*(.*)?$`),
   );
   if (match) {
     const type = match[1] || "code";

src/core/lib/partition-cell-options.ts

@@ -353,6 +353,7 @@ export const kLangCommentChars: Record<string, string | [string, string]> = {
   ojs: "//",
   apl: "⍝",
   ocaml: ["(*", "*)"],
+  q: "/",
   rust: "//",
 };
 

src/resources/filters/common/log.lua

@@ -25,7 +25,7 @@ function warn(message, offset)
 end
 
 function error(message, offset)
-  io.stderr:write(lunacolors.red("ERROR (" .. caller_info(offset) .. ") " .. message .. "\n"))
+  io.stderr:write(lunacolors.red(("ERROR (%s) %s\n"):format(caller_info(offset), message)))
 end
 
 function fatal(message, offset)

src/resources/filters/modules/constants.lua

@@ -116,6 +116,7 @@ local kLangCommentChars = {
   powershell = {"#"},
   psql = {"--"},
   python = {"#"},
+  q = {"/"},
   r = {"#"},
   ruby = {"#"},
   rust = {"//"},

src/resources/jupyter/notebook.py

@@ -875,6 +875,7 @@ def nb_language_comment_chars(lang):
         haskell="--",
         dot="//",
         apl="⍝",
+        q = "/",
         ocaml=["(*", "*)"],
     )
     if lang in langs:

src/resources/rmd/hooks.R

@@ -1046,6 +1046,7 @@ engine_comment_chars <- function(engine) {
     dot = "//",
     apl = "\u235D",
     ocaml = c("(*", "*)"),
+    q = "/",
     rust = "//"
   )
   comment_chars[[engine]] %||% "#"

tests/docs/percent-format/false-positive-raw.py

@@ -0,0 +1,6 @@
+# This file contains the text "raw]" but is NOT a percent script
+# It should test that the regex doesn't match "raw]" anywhere in the file
+# For example: handle raw] data or [raw] text in comments
+
+def process():
+    return "raw]"

tests/docs/percent-format/not-percent.py

@@ -0,0 +1,7 @@
+# This is a regular Python file, not a percent script
+# It has no special cell markers
+
+def hello():
+    print("Hello")
+
+hello()

tests/docs/percent-format/not-percent.q

@@ -0,0 +1,5 @@
+/ This is a regular q file, not a percent script
+/ It has no special cell markers
+
+hello: {[] "Hello"}
+hello[]

tests/docs/percent-format/test-julia.jl

@@ -0,0 +1,11 @@
+# %% [markdown]
+# This is a markdown cell
+
+# %%
+println("Hello from Julia")
+
+# %% [markdown]
+# Another markdown cell
+
+# %%
+x = 1 + 1

tests/docs/percent-format/test-markdown-not-first.py

@@ -0,0 +1,13 @@
+# ---
+# jupyter:
+#   kernelspec:
+#     display_name: Python 3
+#     language: python
+#     name: python3
+# ---
+
+# %%
+print("Code first")
+
+# %% [markdown]
+# This markdown cell is not at the start of the file

tests/docs/percent-format/test-python.py

@@ -0,0 +1,11 @@
+# %% [markdown]
+# This is a markdown cell
+
+# %%
+print("Hello from Python")
+
+# %% [markdown]
+# Another markdown cell
+
+# %%
+x = 1 + 1

tests/docs/percent-format/test-q.q

@@ -0,0 +1,11 @@
+/ %% [markdown]
+/ This is a markdown cell
+
+/ %%
+1+1
+
+/ %% [markdown]
+/ Another markdown cell
+
+/ %%
+x: 42

tests/docs/percent-format/test-r.r

@@ -0,0 +1,11 @@
+# %% [markdown]
+# This is a markdown cell
+
+# %%
+print("Hello from R")
+
+# %% [markdown]
+# Another markdown cell
+
+# %%
+x <- 1 + 1

tests/docs/percent-format/test-raw-cell.py

@@ -0,0 +1,5 @@
+# %% [raw]
+# This is a raw cell
+
+# %%
+print("Hello")