From 864b25807adbf87e37db93fc393e4ce6d652e57c Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Thu, 22 Jan 2026 14:12:46 -0600 Subject: [PATCH 1/7] feat: Add singleton tables (empty primary keys) Implements support for singleton tables - tables with empty primary keys that can hold at most one row. This feature was described in the 2018 DataJoint paper and proposed in issue #113. Syntax: ```python @schema class Config(dj.Lookup): definition = """ --- setting : varchar(100) """ ``` Implementation uses a hidden `_singleton` attribute of type `bool` as the primary key. This attribute is automatically created and excluded from user-facing operations (heading.attributes, fetch, join matching). Closes #113 Co-Authored-By: Claude Opus 4.5 --- src/datajoint/declare.py | 14 ++++- src/datajoint/version.py | 2 +- tests/integration/test_declare.py | 95 +++++++++++++++++++++++++++++++ 3 files changed, 109 insertions(+), 2 deletions(-) diff --git a/src/datajoint/declare.py b/src/datajoint/declare.py index db91bc6cb..5ae7edeb7 100644 --- a/src/datajoint/declare.py +++ b/src/datajoint/declare.py @@ -473,7 +473,19 @@ def declare( attribute_sql.extend(job_metadata_sql) if not primary_key: - raise DataJointError("Table must have a primary key") + # Singleton table: add hidden sentinel attribute + primary_key = ["_singleton"] + singleton_comment = ":bool:singleton primary key" + sql_type = adapter.core_type_to_sql("bool") + singleton_sql = adapter.format_column_definition( + name="_singleton", + sql_type=sql_type, + nullable=False, + default="NOT NULL DEFAULT 1", + comment=singleton_comment, + ) + attribute_sql.insert(0, singleton_sql) + column_comments["_singleton"] = singleton_comment pre_ddl = [] # DDL to run BEFORE CREATE TABLE (e.g., CREATE TYPE for enums) post_ddl = [] # DDL to run AFTER CREATE TABLE (e.g., COMMENT ON) diff --git a/src/datajoint/version.py b/src/datajoint/version.py index 6c722021c..1af370421 100644 --- a/src/datajoint/version.py +++ b/src/datajoint/version.py @@ -1,4 +1,4 @@ # version bump auto managed by Github Actions: # label_prs.yaml(prep), release.yaml(bump), post_release.yaml(edit) # manually set this version will be eventually overwritten by the above actions -__version__ = "2.1.0a2" +__version__ = "2.1.0a3" diff --git a/tests/integration/test_declare.py b/tests/integration/test_declare.py index 36f7b74a3..e0beeb77f 100644 --- a/tests/integration/test_declare.py +++ b/tests/integration/test_declare.py @@ -368,3 +368,98 @@ class Table_With_Underscores(dj.Manual): schema_any(TableNoUnderscores) with pytest.raises(dj.DataJointError, match="must be alphanumeric in CamelCase"): schema_any(Table_With_Underscores) + + +class TestSingletonTables: + """Tests for singleton tables (empty primary keys).""" + + def test_singleton_declaration(self, schema_any): + """Singleton table creates correctly with hidden _singleton attribute.""" + + @schema_any + class Config(dj.Lookup): + definition = """ + # Global configuration + --- + setting : varchar(100) + """ + + # Access attributes first to trigger lazy loading from database + visible_attrs = Config.heading.attributes + all_attrs = Config.heading._attributes + + # Table should exist and have _singleton as hidden PK + assert "_singleton" in all_attrs + assert "_singleton" not in visible_attrs + assert Config.heading.primary_key == [] # Visible PK is empty for singleton + + def test_singleton_insert_and_fetch(self, schema_any): + """Insert and fetch work without specifying _singleton.""" + + @schema_any + class Settings(dj.Lookup): + definition = """ + --- + value : int32 + """ + + # Insert without specifying _singleton + Settings.insert1({"value": 42}) + + # Fetch should work + result = Settings.fetch1() + assert result["value"] == 42 + assert "_singleton" not in result # Hidden attribute excluded + + def test_singleton_uniqueness(self, schema_any): + """Second insert raises DuplicateError.""" + + @schema_any + class SingleValue(dj.Lookup): + definition = """ + --- + data : varchar(50) + """ + + SingleValue.insert1({"data": "first"}) + + # Second insert should fail + with pytest.raises(dj.errors.DuplicateError): + SingleValue.insert1({"data": "second"}) + + def test_singleton_with_multiple_attributes(self, schema_any): + """Singleton table with multiple secondary attributes.""" + + @schema_any + class PipelineConfig(dj.Lookup): + definition = """ + # Pipeline configuration singleton + --- + version : varchar(20) + max_workers : int32 + debug_mode : bool + """ + + PipelineConfig.insert1( + {"version": "1.0.0", "max_workers": 4, "debug_mode": False} + ) + + result = PipelineConfig.fetch1() + assert result["version"] == "1.0.0" + assert result["max_workers"] == 4 + assert result["debug_mode"] == 0 # bool stored as tinyint + + def test_singleton_describe(self, schema_any): + """Describe should show the singleton nature.""" + + @schema_any + class Metadata(dj.Lookup): + definition = """ + --- + info : varchar(255) + """ + + description = Metadata.describe() + # Description should show just the secondary attribute + assert "info" in description + # _singleton is hidden, implementation detail From 996e820f09b000911ffce221d5771bac1c374694 Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Thu, 22 Jan 2026 14:57:54 -0600 Subject: [PATCH 2/7] fix: PostgreSQL compatibility for singleton tables Add boolean_true_literal adapter property to generate correct DEFAULT value for the hidden _singleton attribute: - MySQL: DEFAULT 1 (bool maps to tinyint) - PostgreSQL: DEFAULT TRUE (native boolean) Co-Authored-By: Claude Opus 4.5 --- src/datajoint/adapters/base.py | 15 +++++++++++++++ src/datajoint/adapters/postgres.py | 14 ++++++++++++++ src/datajoint/declare.py | 3 ++- 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/src/datajoint/adapters/base.py b/src/datajoint/adapters/base.py index 35b32ed5f..efdd0a542 100644 --- a/src/datajoint/adapters/base.py +++ b/src/datajoint/adapters/base.py @@ -563,6 +563,21 @@ def supports_inline_indexes(self) -> bool: """ return True # Default for MySQL, override in PostgreSQL + @property + def boolean_true_literal(self) -> str: + """ + Return the SQL literal for boolean TRUE. + + MySQL uses 1 (since bool maps to tinyint). + PostgreSQL uses TRUE (native boolean type). + + Returns + ------- + str + SQL literal for boolean true value. + """ + return "1" # Default for MySQL, override in PostgreSQL + def create_index_ddl( self, full_table_name: str, diff --git a/src/datajoint/adapters/postgres.py b/src/datajoint/adapters/postgres.py index 16455e531..922eb68d8 100644 --- a/src/datajoint/adapters/postgres.py +++ b/src/datajoint/adapters/postgres.py @@ -710,6 +710,20 @@ def supports_inline_indexes(self) -> bool: """ return False + @property + def boolean_true_literal(self) -> str: + """ + Return the SQL literal for boolean TRUE. + + PostgreSQL uses native boolean type with TRUE literal. + + Returns + ------- + str + SQL literal for boolean true value. + """ + return "TRUE" + # ========================================================================= # Introspection # ========================================================================= diff --git a/src/datajoint/declare.py b/src/datajoint/declare.py index 5ae7edeb7..c6501ea3a 100644 --- a/src/datajoint/declare.py +++ b/src/datajoint/declare.py @@ -477,11 +477,12 @@ def declare( primary_key = ["_singleton"] singleton_comment = ":bool:singleton primary key" sql_type = adapter.core_type_to_sql("bool") + bool_literal = adapter.boolean_true_literal singleton_sql = adapter.format_column_definition( name="_singleton", sql_type=sql_type, nullable=False, - default="NOT NULL DEFAULT 1", + default=f"NOT NULL DEFAULT {bool_literal}", comment=singleton_comment, ) attribute_sql.insert(0, singleton_sql) From 25354ad76f794933ea951cf5b1207bfb7aeb9987 Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Thu, 22 Jan 2026 15:22:49 -0600 Subject: [PATCH 3/7] style: Format test file with ruff Co-Authored-By: Claude Opus 4.5 --- tests/integration/test_declare.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/integration/test_declare.py b/tests/integration/test_declare.py index e0beeb77f..d38583cfd 100644 --- a/tests/integration/test_declare.py +++ b/tests/integration/test_declare.py @@ -440,9 +440,7 @@ class PipelineConfig(dj.Lookup): debug_mode : bool """ - PipelineConfig.insert1( - {"version": "1.0.0", "max_workers": 4, "debug_mode": False} - ) + PipelineConfig.insert1({"version": "1.0.0", "max_workers": 4, "debug_mode": False}) result = PipelineConfig.fetch1() assert result["version"] == "1.0.0" From 94fa4a8b96360d11fd0479552e7a4b64ded0c4e1 Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Thu, 22 Jan 2026 15:31:01 -0600 Subject: [PATCH 4/7] chore: Bump version to 2.1.0a4 Co-Authored-By: Claude Opus 4.5 --- src/datajoint/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/datajoint/version.py b/src/datajoint/version.py index 1af370421..4519818c3 100644 --- a/src/datajoint/version.py +++ b/src/datajoint/version.py @@ -1,4 +1,4 @@ # version bump auto managed by Github Actions: # label_prs.yaml(prep), release.yaml(bump), post_release.yaml(edit) # manually set this version will be eventually overwritten by the above actions -__version__ = "2.1.0a3" +__version__ = "2.1.0a4" From d3c7afb8de716041c30f3fdfbc3ff0ef29fa2445 Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Thu, 22 Jan 2026 22:22:33 -0600 Subject: [PATCH 5/7] fix: Backend-agnostic JSON path expressions - Updated translate_attribute() to accept optional adapter parameter - PostgreSQL adapter's json_path_expr() now handles array notation and type casting - Pass adapter to translate_attribute() in condition.py, declare.py, expression.py This enables basic JSON operations to work on both MySQL and PostgreSQL. Co-Authored-By: Claude Opus 4.5 --- src/datajoint/adapters/postgres.py | 33 ++++++++++++++++++++++++------ src/datajoint/condition.py | 27 ++++++++++++++++-------- src/datajoint/declare.py | 2 +- src/datajoint/expression.py | 3 ++- 4 files changed, 49 insertions(+), 16 deletions(-) diff --git a/src/datajoint/adapters/postgres.py b/src/datajoint/adapters/postgres.py index 922eb68d8..f82dd5931 100644 --- a/src/datajoint/adapters/postgres.py +++ b/src/datajoint/adapters/postgres.py @@ -1004,12 +1004,12 @@ def json_path_expr(self, column: str, path: str, return_type: str | None = None) path : str JSON path (e.g., 'field' or 'nested.field'). return_type : str, optional - Return type specification (not used in PostgreSQL jsonb_extract_path_text). + Return type specification for casting (e.g., 'float', 'decimal(10,2)'). Returns ------- str - PostgreSQL jsonb_extract_path_text() expression. + PostgreSQL jsonb_extract_path_text() expression, with optional cast. Examples -------- @@ -1017,13 +1017,34 @@ def json_path_expr(self, column: str, path: str, return_type: str | None = None) 'jsonb_extract_path_text("data", \\'field\\')' >>> adapter.json_path_expr('data', 'nested.field') 'jsonb_extract_path_text("data", \\'nested\\', \\'field\\')' + >>> adapter.json_path_expr('data', 'value', 'float') + 'jsonb_extract_path_text("data", \\'value\\')::float' """ quoted_col = self.quote_identifier(column) - # Split path by '.' for nested access - path_parts = path.split(".") + # Split path by '.' for nested access, handling array notation + path_parts = [] + for part in path.split("."): + # Handle array access like field[0] + if "[" in part: + base, rest = part.split("[", 1) + path_parts.append(base) + # Extract array indices + indices = rest.rstrip("]").split("][") + path_parts.extend(indices) + else: + path_parts.append(part) path_args = ", ".join(f"'{part}'" for part in path_parts) - # Note: PostgreSQL jsonb_extract_path_text doesn't use return type parameter - return f"jsonb_extract_path_text({quoted_col}, {path_args})" + expr = f"jsonb_extract_path_text({quoted_col}, {path_args})" + # Add cast if return type specified + if return_type: + # Map DataJoint types to PostgreSQL types + pg_type = return_type.lower() + if pg_type in ("unsigned", "signed"): + pg_type = "integer" + elif pg_type == "double": + pg_type = "double precision" + expr = f"({expr})::{pg_type}" + return expr def translate_expression(self, expr: str) -> str: """ diff --git a/src/datajoint/condition.py b/src/datajoint/condition.py index 62550f0d6..69a89f6b7 100644 --- a/src/datajoint/condition.py +++ b/src/datajoint/condition.py @@ -31,7 +31,7 @@ JSON_PATTERN = re.compile(r"^(?P\w+)(\.(?P[\w.*\[\]]+))?(:(?P[\w(,\s)]+))?$") -def translate_attribute(key: str) -> tuple[dict | None, str]: +def translate_attribute(key: str, adapter=None) -> tuple[dict | None, str]: """ Translate an attribute key, handling JSON path notation. @@ -39,6 +39,9 @@ def translate_attribute(key: str) -> tuple[dict | None, str]: ---------- key : str Attribute name, optionally with JSON path (e.g., ``"attr.path.field"``). + adapter : DatabaseAdapter, optional + Database adapter for backend-specific SQL generation. + If not provided, uses MySQL syntax for backward compatibility. Returns ------- @@ -53,9 +56,14 @@ def translate_attribute(key: str) -> tuple[dict | None, str]: if match["path"] is None: return match, match["attr"] else: - return match, "json_value(`{}`, _utf8mb4'$.{}'{})".format( - *[((f" returning {v}" if k == "type" else v) if v else "") for k, v in match.items()] - ) + # Use adapter's json_path_expr if available, otherwise fall back to MySQL syntax + if adapter is not None: + return match, adapter.json_path_expr(match["attr"], match["path"], match["type"]) + else: + # Legacy MySQL syntax for backward compatibility + return match, "json_value(`{}`, _utf8mb4'$.{}'{})".format( + *[((f" returning {v}" if k == "type" else v) if v else "") for k, v in match.items()] + ) class PromiscuousOperand: @@ -306,14 +314,17 @@ def make_condition( def prep_value(k, v): """prepare SQL condition""" - key_match, k = translate_attribute(k) - if key_match["path"] is None: + key_match, k = translate_attribute(k, adapter) + is_json_path = key_match is not None and key_match.get("path") is not None + has_explicit_type = key_match is not None and key_match.get("type") is not None + + if not is_json_path: k = adapter.quote_identifier(k) - if query_expression.heading[key_match["attr"]].json and key_match["path"] is not None and isinstance(v, dict): + if is_json_path and isinstance(v, dict): return f"{k}='{json.dumps(v)}'" if v is None: return f"{k} IS NULL" - if query_expression.heading[key_match["attr"]].uuid: + if key_match is not None and query_expression.heading[key_match["attr"]].uuid: if not isinstance(v, uuid.UUID): try: v = uuid.UUID(v) diff --git a/src/datajoint/declare.py b/src/datajoint/declare.py index c6501ea3a..696accf78 100644 --- a/src/datajoint/declare.py +++ b/src/datajoint/declare.py @@ -755,7 +755,7 @@ def compile_index(line: str, index_sql: list[str], adapter) -> None: """ def format_attribute(attr): - match, attr = translate_attribute(attr) + match, attr = translate_attribute(attr, adapter) if match is None: return attr if match["path"] is None: diff --git a/src/datajoint/expression.py b/src/datajoint/expression.py index b09e2fa78..6e881e11f 100644 --- a/src/datajoint/expression.py +++ b/src/datajoint/expression.py @@ -457,7 +457,8 @@ def proj(self, *attributes, **named_attributes): from other attributes available before the projection. Each attribute name can only be used once. """ - named_attributes = {k: translate_attribute(v)[1] for k, v in named_attributes.items()} + adapter = self.connection.adapter if hasattr(self, 'connection') and self.connection else None + named_attributes = {k: translate_attribute(v, adapter)[1] for k, v in named_attributes.items()} # new attributes in parentheses are included again with the new name without removing original duplication_pattern = re.compile(rf"^\s*\(\s*(?!{'|'.join(CONSTANT_LITERALS)})(?P[a-zA-Z_]\w*)\s*\)\s*$") # attributes without parentheses renamed From 215bc9c4d08940c3eea4d4bb5db8425fc9e30828 Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Thu, 22 Jan 2026 22:28:06 -0600 Subject: [PATCH 6/7] chore: Bump version to 2.1.0a5 Co-Authored-By: Claude Opus 4.5 --- src/datajoint/condition.py | 1 - src/datajoint/expression.py | 2 +- src/datajoint/version.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/datajoint/condition.py b/src/datajoint/condition.py index 69a89f6b7..0335d6adb 100644 --- a/src/datajoint/condition.py +++ b/src/datajoint/condition.py @@ -316,7 +316,6 @@ def prep_value(k, v): """prepare SQL condition""" key_match, k = translate_attribute(k, adapter) is_json_path = key_match is not None and key_match.get("path") is not None - has_explicit_type = key_match is not None and key_match.get("type") is not None if not is_json_path: k = adapter.quote_identifier(k) diff --git a/src/datajoint/expression.py b/src/datajoint/expression.py index 6e881e11f..6decaf336 100644 --- a/src/datajoint/expression.py +++ b/src/datajoint/expression.py @@ -457,7 +457,7 @@ def proj(self, *attributes, **named_attributes): from other attributes available before the projection. Each attribute name can only be used once. """ - adapter = self.connection.adapter if hasattr(self, 'connection') and self.connection else None + adapter = self.connection.adapter if hasattr(self, "connection") and self.connection else None named_attributes = {k: translate_attribute(v, adapter)[1] for k, v in named_attributes.items()} # new attributes in parentheses are included again with the new name without removing original duplication_pattern = re.compile(rf"^\s*\(\s*(?!{'|'.join(CONSTANT_LITERALS)})(?P[a-zA-Z_]\w*)\s*\)\s*$") diff --git a/src/datajoint/version.py b/src/datajoint/version.py index 4519818c3..2ffb3afa8 100644 --- a/src/datajoint/version.py +++ b/src/datajoint/version.py @@ -1,4 +1,4 @@ # version bump auto managed by Github Actions: # label_prs.yaml(prep), release.yaml(bump), post_release.yaml(edit) # manually set this version will be eventually overwritten by the above actions -__version__ = "2.1.0a4" +__version__ = "2.1.0a5" From 5010b857377c46dfd3a24680c715105eb8272ad1 Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Fri, 23 Jan 2026 10:17:36 -0600 Subject: [PATCH 7/7] refactor: Remove boolean_true_literal, use TRUE for both backends MySQL accepts TRUE as a boolean literal (alias for 1), so we can use TRUE universally instead of having backend-specific literals. Co-Authored-By: Claude Opus 4.5 --- src/datajoint/adapters/base.py | 15 --------------- src/datajoint/adapters/postgres.py | 14 -------------- src/datajoint/declare.py | 3 +-- 3 files changed, 1 insertion(+), 31 deletions(-) diff --git a/src/datajoint/adapters/base.py b/src/datajoint/adapters/base.py index efdd0a542..35b32ed5f 100644 --- a/src/datajoint/adapters/base.py +++ b/src/datajoint/adapters/base.py @@ -563,21 +563,6 @@ def supports_inline_indexes(self) -> bool: """ return True # Default for MySQL, override in PostgreSQL - @property - def boolean_true_literal(self) -> str: - """ - Return the SQL literal for boolean TRUE. - - MySQL uses 1 (since bool maps to tinyint). - PostgreSQL uses TRUE (native boolean type). - - Returns - ------- - str - SQL literal for boolean true value. - """ - return "1" # Default for MySQL, override in PostgreSQL - def create_index_ddl( self, full_table_name: str, diff --git a/src/datajoint/adapters/postgres.py b/src/datajoint/adapters/postgres.py index f82dd5931..12fecae6a 100644 --- a/src/datajoint/adapters/postgres.py +++ b/src/datajoint/adapters/postgres.py @@ -710,20 +710,6 @@ def supports_inline_indexes(self) -> bool: """ return False - @property - def boolean_true_literal(self) -> str: - """ - Return the SQL literal for boolean TRUE. - - PostgreSQL uses native boolean type with TRUE literal. - - Returns - ------- - str - SQL literal for boolean true value. - """ - return "TRUE" - # ========================================================================= # Introspection # ========================================================================= diff --git a/src/datajoint/declare.py b/src/datajoint/declare.py index 696accf78..375daa07e 100644 --- a/src/datajoint/declare.py +++ b/src/datajoint/declare.py @@ -477,12 +477,11 @@ def declare( primary_key = ["_singleton"] singleton_comment = ":bool:singleton primary key" sql_type = adapter.core_type_to_sql("bool") - bool_literal = adapter.boolean_true_literal singleton_sql = adapter.format_column_definition( name="_singleton", sql_type=sql_type, nullable=False, - default=f"NOT NULL DEFAULT {bool_literal}", + default="NOT NULL DEFAULT TRUE", comment=singleton_comment, ) attribute_sql.insert(0, singleton_sql)