From 6a86172dfd4cc5b7df89dc0d01d31de421e2ad6f Mon Sep 17 00:00:00 2001 From: Alex Soffronow-Pagonidis Date: Fri, 10 Apr 2026 15:46:05 +0200 Subject: [PATCH 1/2] add support for migrations --- .github/workflows/ef-cli-smoke.yml | 95 +++ .../ClickHouseAnnotationCodeGenerator.cs | 46 ++ .../Internal/ClickHouseCodeGenerator.cs | 30 + .../Internal/ClickHouseDesignTimeServices.cs | 22 + .../EFCore.ClickHouse.csproj | 1 + .../ClickHouseEntityTypeBuilderExtensions.cs | 88 +++ .../ClickHouseEntityTypeExtensions.cs | 132 ++++ .../ClickHouseIndexBuilderExtensions.cs | 25 + .../Extensions/ClickHouseIndexExtensions.cs | 31 + .../ClickHousePropertyBuilderExtensions.cs | 46 ++ .../ClickHousePropertyExtensions.cs | 31 + .../ClickHouseServiceCollectionExtensions.cs | 6 + .../Internal/ClickHouseModelValidator.cs | 63 ++ ...kHouseAggregatingMergeTreeEngineBuilder.cs | 48 ++ ...ckHouseCollapsingMergeTreeEngineBuilder.cs | 50 ++ .../Builders/ClickHouseEngineBuilder.cs | 61 ++ ...lickHouseGraphiteMergeTreeEngineBuilder.cs | 50 ++ .../ClickHouseMergeTreeEngineBuilder.cs | 48 ++ ...ickHouseReplacingMergeTreeEngineBuilder.cs | 54 ++ .../Builders/ClickHouseSimpleEngineBuilder.cs | 40 ++ ...ClickHouseSummingMergeTreeEngineBuilder.cs | 51 ++ ...rsionedCollapsingMergeTreeEngineBuilder.cs | 52 ++ .../ClickHouseConventionSetBuilder.cs | 28 +- .../ClickHouseDefaultEngineConvention.cs | 60 ++ .../Internal/ClickHouseAnnotationNames.cs | 39 ++ .../Internal/ClickHouseAnnotationProvider.cs | 43 ++ .../ClickHouseMigrationsSqlGenerator.cs | 567 ++++++++++++++++++ .../Internal/ClickHouseHistoryRepository.cs | 60 ++ .../ClickHouseMigrationDatabaseLock.cs | 20 + .../ClickHouseMigrationsAnnotationProvider.cs | 11 + .../ClickHouseCreateDatabaseOperation.cs | 8 + .../ClickHouseDropDatabaseOperation.cs | 8 + .../EFCore.ClickHouse.DesignSmoke.csproj | 15 + test/EFCore.ClickHouse.DesignSmoke/Program.cs | 2 + .../SmokeDbContext.cs | 67 +++ .../EngineConfigurationTests.cs | 368 ++++++++++++ .../EnsureCreatedTests.cs | 241 ++++++++ .../MigrationSqlGeneratorTests.cs | 559 +++++++++++++++++ 38 files changed, 3164 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/ef-cli-smoke.yml create mode 100644 src/EFCore.ClickHouse/Design/Internal/ClickHouseAnnotationCodeGenerator.cs create mode 100644 src/EFCore.ClickHouse/Design/Internal/ClickHouseCodeGenerator.cs create mode 100644 src/EFCore.ClickHouse/Design/Internal/ClickHouseDesignTimeServices.cs create mode 100644 src/EFCore.ClickHouse/Extensions/ClickHouseEntityTypeBuilderExtensions.cs create mode 100644 src/EFCore.ClickHouse/Extensions/ClickHouseEntityTypeExtensions.cs create mode 100644 src/EFCore.ClickHouse/Extensions/ClickHouseIndexBuilderExtensions.cs create mode 100644 src/EFCore.ClickHouse/Extensions/ClickHouseIndexExtensions.cs create mode 100644 src/EFCore.ClickHouse/Extensions/ClickHousePropertyBuilderExtensions.cs create mode 100644 src/EFCore.ClickHouse/Extensions/ClickHousePropertyExtensions.cs create mode 100644 src/EFCore.ClickHouse/Metadata/Builders/ClickHouseAggregatingMergeTreeEngineBuilder.cs create mode 100644 src/EFCore.ClickHouse/Metadata/Builders/ClickHouseCollapsingMergeTreeEngineBuilder.cs create mode 100644 src/EFCore.ClickHouse/Metadata/Builders/ClickHouseEngineBuilder.cs create mode 100644 src/EFCore.ClickHouse/Metadata/Builders/ClickHouseGraphiteMergeTreeEngineBuilder.cs create mode 100644 src/EFCore.ClickHouse/Metadata/Builders/ClickHouseMergeTreeEngineBuilder.cs create mode 100644 src/EFCore.ClickHouse/Metadata/Builders/ClickHouseReplacingMergeTreeEngineBuilder.cs create mode 100644 src/EFCore.ClickHouse/Metadata/Builders/ClickHouseSimpleEngineBuilder.cs create mode 100644 src/EFCore.ClickHouse/Metadata/Builders/ClickHouseSummingMergeTreeEngineBuilder.cs create mode 100644 src/EFCore.ClickHouse/Metadata/Builders/ClickHouseVersionedCollapsingMergeTreeEngineBuilder.cs create mode 100644 src/EFCore.ClickHouse/Metadata/Conventions/ClickHouseDefaultEngineConvention.cs create mode 100644 src/EFCore.ClickHouse/Migrations/ClickHouseMigrationsSqlGenerator.cs create mode 100644 src/EFCore.ClickHouse/Migrations/Internal/ClickHouseHistoryRepository.cs create mode 100644 src/EFCore.ClickHouse/Migrations/Internal/ClickHouseMigrationDatabaseLock.cs create mode 100644 src/EFCore.ClickHouse/Migrations/Internal/ClickHouseMigrationsAnnotationProvider.cs create mode 100644 src/EFCore.ClickHouse/Migrations/Operations/ClickHouseCreateDatabaseOperation.cs create mode 100644 src/EFCore.ClickHouse/Migrations/Operations/ClickHouseDropDatabaseOperation.cs create mode 100644 test/EFCore.ClickHouse.DesignSmoke/EFCore.ClickHouse.DesignSmoke.csproj create mode 100644 test/EFCore.ClickHouse.DesignSmoke/Program.cs create mode 100644 test/EFCore.ClickHouse.DesignSmoke/SmokeDbContext.cs create mode 100644 test/EFCore.ClickHouse.Tests/EngineConfigurationTests.cs create mode 100644 test/EFCore.ClickHouse.Tests/EnsureCreatedTests.cs create mode 100644 test/EFCore.ClickHouse.Tests/MigrationSqlGeneratorTests.cs diff --git a/.github/workflows/ef-cli-smoke.yml b/.github/workflows/ef-cli-smoke.yml new file mode 100644 index 0000000..3250992 --- /dev/null +++ b/.github/workflows/ef-cli-smoke.yml @@ -0,0 +1,95 @@ +name: EF CLI Smoke + +on: + push: + branches: [main] + paths-ignore: + - "**/*.md" + pull_request: + branches: [main] + +jobs: + ef-cli-smoke: + name: dotnet-ef smoke test + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Setup .NET + uses: actions/setup-dotnet@v5 + with: + dotnet-version: 10.0.x + + - name: Install dotnet-ef + run: dotnet tool install --global dotnet-ef + + - name: Restore & Build + run: dotnet build test/EFCore.ClickHouse.DesignSmoke/EFCore.ClickHouse.DesignSmoke.csproj + + - name: Add migration + run: | + dotnet-ef migrations add InitialCreate \ + --project test/EFCore.ClickHouse.DesignSmoke \ + --startup-project test/EFCore.ClickHouse.DesignSmoke + + - name: Verify migration contains ClickHouse annotations + run: | + MIGRATION_FILE=$(find test/EFCore.ClickHouse.DesignSmoke/Migrations -name '*_InitialCreate.cs' ! -name '*.Designer.cs') + echo "Checking $MIGRATION_FILE" + + grep -F 'ClickHouse:Engine' "$MIGRATION_FILE" + grep -F 'ClickHouse:OrderBy' "$MIGRATION_FILE" + grep -F 'ClickHouse:PartitionBy' "$MIGRATION_FILE" + grep -F 'ClickHouse:PrimaryKey' "$MIGRATION_FILE" + grep -F 'ClickHouse:SampleBy' "$MIGRATION_FILE" + grep -F 'ClickHouse:Setting:index_granularity' "$MIGRATION_FILE" + grep -F 'ClickHouse:Ttl' "$MIGRATION_FILE" + grep -F 'ClickHouse:ColumnCodec' "$MIGRATION_FILE" + grep -F 'ClickHouse:ColumnTtl' "$MIGRATION_FILE" + grep -F 'ClickHouse:ColumnComment' "$MIGRATION_FILE" + grep -F 'ClickHouse:SkippingIndex:Type' "$MIGRATION_FILE" + grep -F 'ClickHouse:SkippingIndex:Granularity' "$MIGRATION_FILE" + echo "All ClickHouse annotations present in migration." + + - name: Generate SQL script + run: | + dotnet-ef migrations script \ + --project test/EFCore.ClickHouse.DesignSmoke \ + --startup-project test/EFCore.ClickHouse.DesignSmoke \ + > migration.sql + cat migration.sql + + - name: Verify SQL script content + run: | + # Engine and clauses + grep -F 'ENGINE = ReplacingMergeTree' migration.sql + grep -F 'ORDER BY' migration.sql + grep -F 'PARTITION BY' migration.sql + grep -F 'PRIMARY KEY' migration.sql + grep -F 'SAMPLE BY' migration.sql + grep -F 'TTL Timestamp + INTERVAL 1 YEAR' migration.sql + grep -F 'SETTINGS index_granularity = 4096' migration.sql + grep -F 'ADD INDEX' migration.sql + grep -F 'CODEC(Delta, ZSTD)' migration.sql + + # Simple engine without parentheses + grep -F 'ENGINE = Memory' migration.sql + ! grep -F 'ENGINE = Memory()' migration.sql + + # No transaction wrapping (ClickHouse does not support transactions) + ! grep -F 'START TRANSACTION' migration.sql + ! grep -F 'COMMIT' migration.sql + + echo "SQL script content verified." + + - name: Verify idempotent script is rejected + run: | + if dotnet-ef migrations script --idempotent \ + --project test/EFCore.ClickHouse.DesignSmoke \ + --startup-project test/EFCore.ClickHouse.DesignSmoke 2>&1; then + echo "ERROR: --idempotent should have failed" + exit 1 + else + echo "Idempotent script correctly rejected." + fi diff --git a/src/EFCore.ClickHouse/Design/Internal/ClickHouseAnnotationCodeGenerator.cs b/src/EFCore.ClickHouse/Design/Internal/ClickHouseAnnotationCodeGenerator.cs new file mode 100644 index 0000000..e955688 --- /dev/null +++ b/src/EFCore.ClickHouse/Design/Internal/ClickHouseAnnotationCodeGenerator.cs @@ -0,0 +1,46 @@ +using ClickHouse.EntityFrameworkCore.Metadata.Internal; +using Microsoft.EntityFrameworkCore.Design; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; + +namespace ClickHouse.EntityFrameworkCore.Design.Internal; + +public class ClickHouseAnnotationCodeGenerator : AnnotationCodeGenerator +{ + public ClickHouseAnnotationCodeGenerator(AnnotationCodeGeneratorDependencies dependencies) + : base(dependencies) + { + } + + protected override bool IsHandledByConvention(IModel model, IAnnotation annotation) + { + if (annotation.Name.StartsWith(ClickHouseAnnotationNames.Prefix, StringComparison.Ordinal)) + return false; + + return base.IsHandledByConvention(model, annotation); + } + + protected override bool IsHandledByConvention(IEntityType entityType, IAnnotation annotation) + { + if (annotation.Name.StartsWith(ClickHouseAnnotationNames.Prefix, StringComparison.Ordinal)) + return false; + + return base.IsHandledByConvention(entityType, annotation); + } + + protected override bool IsHandledByConvention(IProperty property, IAnnotation annotation) + { + if (annotation.Name.StartsWith(ClickHouseAnnotationNames.Prefix, StringComparison.Ordinal)) + return false; + + return base.IsHandledByConvention(property, annotation); + } + + protected override bool IsHandledByConvention(IIndex index, IAnnotation annotation) + { + if (annotation.Name.StartsWith(ClickHouseAnnotationNames.Prefix, StringComparison.Ordinal)) + return false; + + return base.IsHandledByConvention(index, annotation); + } +} diff --git a/src/EFCore.ClickHouse/Design/Internal/ClickHouseCodeGenerator.cs b/src/EFCore.ClickHouse/Design/Internal/ClickHouseCodeGenerator.cs new file mode 100644 index 0000000..e2c591f --- /dev/null +++ b/src/EFCore.ClickHouse/Design/Internal/ClickHouseCodeGenerator.cs @@ -0,0 +1,30 @@ +using ClickHouse.EntityFrameworkCore.Extensions; +using ClickHouse.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Design; +using Microsoft.EntityFrameworkCore.Scaffolding; +using System.Reflection; + +namespace ClickHouse.EntityFrameworkCore.Design.Internal; + +public class ClickHouseCodeGenerator : ProviderCodeGenerator +{ + private static readonly MethodInfo UseClickHouseMethodInfo + = typeof(ClickHouseDbContextOptionsBuilderExtensions).GetRuntimeMethod( + nameof(ClickHouseDbContextOptionsBuilderExtensions.UseClickHouse), + [typeof(DbContextOptionsBuilder), typeof(string), typeof(Action)])!; + + public ClickHouseCodeGenerator(ProviderCodeGeneratorDependencies dependencies) + : base(dependencies) + { + } + + public override MethodCallCodeFragment GenerateUseProvider( + string connectionString, + MethodCallCodeFragment? providerOptions) + => new( + UseClickHouseMethodInfo, + providerOptions is null + ? [connectionString] + : [connectionString, new NestedClosureCodeFragment("x", providerOptions)]); +} diff --git a/src/EFCore.ClickHouse/Design/Internal/ClickHouseDesignTimeServices.cs b/src/EFCore.ClickHouse/Design/Internal/ClickHouseDesignTimeServices.cs new file mode 100644 index 0000000..5b85cd8 --- /dev/null +++ b/src/EFCore.ClickHouse/Design/Internal/ClickHouseDesignTimeServices.cs @@ -0,0 +1,22 @@ +using ClickHouse.EntityFrameworkCore.Extensions; +using Microsoft.EntityFrameworkCore.Design; +using Microsoft.EntityFrameworkCore.Scaffolding; +using Microsoft.Extensions.DependencyInjection; + +[assembly: DesignTimeProviderServices( + "ClickHouse.EntityFrameworkCore.Design.Internal.ClickHouseDesignTimeServices")] + +namespace ClickHouse.EntityFrameworkCore.Design.Internal; + +public class ClickHouseDesignTimeServices : IDesignTimeServices +{ + public void ConfigureDesignTimeServices(IServiceCollection serviceCollection) + { + serviceCollection.AddEntityFrameworkClickHouse(); + + new EntityFrameworkRelationalDesignServicesBuilder(serviceCollection) + .TryAdd() + .TryAdd() + .TryAddCoreServices(); + } +} diff --git a/src/EFCore.ClickHouse/EFCore.ClickHouse.csproj b/src/EFCore.ClickHouse/EFCore.ClickHouse.csproj index f1c819f..5b89340 100644 --- a/src/EFCore.ClickHouse/EFCore.ClickHouse.csproj +++ b/src/EFCore.ClickHouse/EFCore.ClickHouse.csproj @@ -25,6 +25,7 @@ + diff --git a/src/EFCore.ClickHouse/Extensions/ClickHouseEntityTypeBuilderExtensions.cs b/src/EFCore.ClickHouse/Extensions/ClickHouseEntityTypeBuilderExtensions.cs new file mode 100644 index 0000000..1048af6 --- /dev/null +++ b/src/EFCore.ClickHouse/Extensions/ClickHouseEntityTypeBuilderExtensions.cs @@ -0,0 +1,88 @@ +using ClickHouse.EntityFrameworkCore.Metadata.Builders; +using ClickHouse.EntityFrameworkCore.Metadata.Internal; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +namespace ClickHouse.EntityFrameworkCore.Extensions; + +public static class ClickHouseEntityTypeBuilderExtensions +{ + public static ClickHouseMergeTreeEngineBuilder HasMergeTreeEngine(this TableBuilder tableBuilder) + { + ArgumentNullException.ThrowIfNull(tableBuilder); + return new(GetEntityType(tableBuilder)); + } + + public static ClickHouseReplacingMergeTreeEngineBuilder HasReplacingMergeTreeEngine( + this TableBuilder tableBuilder, string? version = null, string? isDeleted = null) + { + ArgumentNullException.ThrowIfNull(tableBuilder); + return new(GetEntityType(tableBuilder), version, isDeleted); + } + + public static ClickHouseSummingMergeTreeEngineBuilder HasSummingMergeTreeEngine( + this TableBuilder tableBuilder, params string[] columns) + { + ArgumentNullException.ThrowIfNull(tableBuilder); + return new(GetEntityType(tableBuilder), columns); + } + + public static ClickHouseAggregatingMergeTreeEngineBuilder HasAggregatingMergeTreeEngine( + this TableBuilder tableBuilder) + { + ArgumentNullException.ThrowIfNull(tableBuilder); + return new(GetEntityType(tableBuilder)); + } + + public static ClickHouseCollapsingMergeTreeEngineBuilder HasCollapsingMergeTreeEngine( + this TableBuilder tableBuilder, string sign) + { + ArgumentNullException.ThrowIfNull(tableBuilder); + ArgumentException.ThrowIfNullOrWhiteSpace(sign); + return new(GetEntityType(tableBuilder), sign); + } + + public static ClickHouseVersionedCollapsingMergeTreeEngineBuilder HasVersionedCollapsingMergeTreeEngine( + this TableBuilder tableBuilder, string sign, string version) + { + ArgumentNullException.ThrowIfNull(tableBuilder); + ArgumentException.ThrowIfNullOrWhiteSpace(sign); + ArgumentException.ThrowIfNullOrWhiteSpace(version); + return new(GetEntityType(tableBuilder), sign, version); + } + + public static ClickHouseGraphiteMergeTreeEngineBuilder HasGraphiteMergeTreeEngine( + this TableBuilder tableBuilder, string configSection) + { + ArgumentNullException.ThrowIfNull(tableBuilder); + ArgumentException.ThrowIfNullOrWhiteSpace(configSection); + return new(GetEntityType(tableBuilder), configSection); + } + + public static ClickHouseSimpleEngineBuilder HasTinyLogEngine(this TableBuilder tableBuilder) + { + ArgumentNullException.ThrowIfNull(tableBuilder); + return new(GetEntityType(tableBuilder), ClickHouseAnnotationNames.TinyLog); + } + + public static ClickHouseSimpleEngineBuilder HasStripeLogEngine(this TableBuilder tableBuilder) + { + ArgumentNullException.ThrowIfNull(tableBuilder); + return new(GetEntityType(tableBuilder), ClickHouseAnnotationNames.StripeLog); + } + + public static ClickHouseSimpleEngineBuilder HasLogEngine(this TableBuilder tableBuilder) + { + ArgumentNullException.ThrowIfNull(tableBuilder); + return new(GetEntityType(tableBuilder), ClickHouseAnnotationNames.Log); + } + + public static ClickHouseSimpleEngineBuilder HasMemoryEngine(this TableBuilder tableBuilder) + { + ArgumentNullException.ThrowIfNull(tableBuilder); + return new(GetEntityType(tableBuilder), ClickHouseAnnotationNames.Memory); + } + + private static IMutableEntityType GetEntityType(TableBuilder tableBuilder) + => (IMutableEntityType)tableBuilder.Metadata; +} diff --git a/src/EFCore.ClickHouse/Extensions/ClickHouseEntityTypeExtensions.cs b/src/EFCore.ClickHouse/Extensions/ClickHouseEntityTypeExtensions.cs new file mode 100644 index 0000000..aef61c4 --- /dev/null +++ b/src/EFCore.ClickHouse/Extensions/ClickHouseEntityTypeExtensions.cs @@ -0,0 +1,132 @@ +using ClickHouse.EntityFrameworkCore.Metadata.Internal; +using Microsoft.EntityFrameworkCore.Metadata; + +namespace ClickHouse.EntityFrameworkCore.Extensions; + +public static class ClickHouseEntityTypeExtensions +{ + // Engine + + public static string? GetEngine(this IReadOnlyEntityType entityType) + => (string?)entityType[ClickHouseAnnotationNames.Engine]; + + public static void SetEngine(this IMutableEntityType entityType, string? engine) + => entityType.SetOrRemoveAnnotation(ClickHouseAnnotationNames.Engine, engine); + + // ORDER BY + + public static string[]? GetOrderBy(this IReadOnlyEntityType entityType) + => (string[]?)entityType[ClickHouseAnnotationNames.OrderBy]; + + public static void SetOrderBy(this IMutableEntityType entityType, string[]? columns) + => entityType.SetOrRemoveAnnotation(ClickHouseAnnotationNames.OrderBy, + columns is { Length: > 0 } ? columns : null); + + // PARTITION BY + + public static string[]? GetPartitionBy(this IReadOnlyEntityType entityType) + => (string[]?)entityType[ClickHouseAnnotationNames.PartitionBy]; + + public static void SetPartitionBy(this IMutableEntityType entityType, string[]? columns) + => entityType.SetOrRemoveAnnotation(ClickHouseAnnotationNames.PartitionBy, + columns is { Length: > 0 } ? columns : null); + + // PRIMARY KEY (ClickHouse structural key, distinct from EF's HasKey) + + public static string[]? GetClickHousePrimaryKey(this IReadOnlyEntityType entityType) + => (string[]?)entityType[ClickHouseAnnotationNames.PrimaryKey]; + + public static void SetClickHousePrimaryKey(this IMutableEntityType entityType, string[]? columns) + => entityType.SetOrRemoveAnnotation(ClickHouseAnnotationNames.PrimaryKey, + columns is { Length: > 0 } ? columns : null); + + // SAMPLE BY + + public static string[]? GetSampleBy(this IReadOnlyEntityType entityType) + => (string[]?)entityType[ClickHouseAnnotationNames.SampleBy]; + + public static void SetSampleBy(this IMutableEntityType entityType, string[]? columns) + => entityType.SetOrRemoveAnnotation(ClickHouseAnnotationNames.SampleBy, + columns is { Length: > 0 } ? columns : null); + + // TTL (table-level) + + public static string? GetTtl(this IReadOnlyEntityType entityType) + => (string?)entityType[ClickHouseAnnotationNames.Ttl]; + + public static void SetTtl(this IMutableEntityType entityType, string? ttlExpression) + => entityType.SetOrRemoveAnnotation(ClickHouseAnnotationNames.Ttl, ttlExpression); + + // ReplacingMergeTree + + public static string? GetReplacingMergeTreeVersion(this IReadOnlyEntityType entityType) + => (string?)entityType[ClickHouseAnnotationNames.ReplacingMergeTreeVersion]; + + public static void SetReplacingMergeTreeVersion(this IMutableEntityType entityType, string? version) + => entityType.SetOrRemoveAnnotation(ClickHouseAnnotationNames.ReplacingMergeTreeVersion, version); + + public static string? GetReplacingMergeTreeIsDeleted(this IReadOnlyEntityType entityType) + => (string?)entityType[ClickHouseAnnotationNames.ReplacingMergeTreeIsDeleted]; + + public static void SetReplacingMergeTreeIsDeleted(this IMutableEntityType entityType, string? isDeleted) + => entityType.SetOrRemoveAnnotation(ClickHouseAnnotationNames.ReplacingMergeTreeIsDeleted, isDeleted); + + // SummingMergeTree + + public static string[]? GetSummingMergeTreeColumns(this IReadOnlyEntityType entityType) + => (string[]?)entityType[ClickHouseAnnotationNames.SummingMergeTreeColumns]; + + public static void SetSummingMergeTreeColumns(this IMutableEntityType entityType, string[]? columns) + => entityType.SetOrRemoveAnnotation(ClickHouseAnnotationNames.SummingMergeTreeColumns, + columns is { Length: > 0 } ? columns : null); + + // CollapsingMergeTree + + public static string? GetCollapsingMergeTreeSign(this IReadOnlyEntityType entityType) + => (string?)entityType[ClickHouseAnnotationNames.CollapsingMergeTreeSign]; + + public static void SetCollapsingMergeTreeSign(this IMutableEntityType entityType, string? sign) + => entityType.SetOrRemoveAnnotation(ClickHouseAnnotationNames.CollapsingMergeTreeSign, sign); + + // VersionedCollapsingMergeTree + + public static string? GetVersionedCollapsingMergeTreeSign(this IReadOnlyEntityType entityType) + => (string?)entityType[ClickHouseAnnotationNames.VersionedCollapsingMergeTreeSign]; + + public static void SetVersionedCollapsingMergeTreeSign(this IMutableEntityType entityType, string? sign) + => entityType.SetOrRemoveAnnotation(ClickHouseAnnotationNames.VersionedCollapsingMergeTreeSign, sign); + + public static string? GetVersionedCollapsingMergeTreeVersion(this IReadOnlyEntityType entityType) + => (string?)entityType[ClickHouseAnnotationNames.VersionedCollapsingMergeTreeVersion]; + + public static void SetVersionedCollapsingMergeTreeVersion(this IMutableEntityType entityType, string? version) + => entityType.SetOrRemoveAnnotation(ClickHouseAnnotationNames.VersionedCollapsingMergeTreeVersion, version); + + // GraphiteMergeTree + + public static string? GetGraphiteMergeTreeConfigSection(this IReadOnlyEntityType entityType) + => (string?)entityType[ClickHouseAnnotationNames.GraphiteMergeTreeConfigSection]; + + public static void SetGraphiteMergeTreeConfigSection(this IMutableEntityType entityType, string? configSection) + => entityType.SetOrRemoveAnnotation(ClickHouseAnnotationNames.GraphiteMergeTreeConfigSection, configSection); + + // Settings (prefix-based key-value) + + public static Dictionary GetSettings(this IReadOnlyEntityType entityType) + { + var settings = new Dictionary(); + foreach (var annotation in entityType.GetAnnotations()) + { + if (annotation.Name.StartsWith(ClickHouseAnnotationNames.SettingPrefix, StringComparison.Ordinal) + && annotation.Value is string value) + { + var key = annotation.Name[ClickHouseAnnotationNames.SettingPrefix.Length..]; + settings[key] = value; + } + } + return settings; + } + + public static void SetSetting(this IMutableEntityType entityType, string settingName, string? value) + => entityType.SetOrRemoveAnnotation(ClickHouseAnnotationNames.SettingPrefix + settingName, value); +} diff --git a/src/EFCore.ClickHouse/Extensions/ClickHouseIndexBuilderExtensions.cs b/src/EFCore.ClickHouse/Extensions/ClickHouseIndexBuilderExtensions.cs new file mode 100644 index 0000000..123bc32 --- /dev/null +++ b/src/EFCore.ClickHouse/Extensions/ClickHouseIndexBuilderExtensions.cs @@ -0,0 +1,25 @@ +using ClickHouse.EntityFrameworkCore.Metadata.Internal; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +namespace ClickHouse.EntityFrameworkCore.Extensions; + +public static class ClickHouseIndexBuilderExtensions +{ + public static IndexBuilder HasSkippingIndexType(this IndexBuilder indexBuilder, string type) + { + indexBuilder.Metadata.SetSkippingIndexType(type); + return indexBuilder; + } + + public static IndexBuilder HasGranularity(this IndexBuilder indexBuilder, int granularity) + { + indexBuilder.Metadata.SetGranularity(granularity); + return indexBuilder; + } + + public static IndexBuilder HasSkippingIndexParams(this IndexBuilder indexBuilder, string parameters) + { + indexBuilder.Metadata.SetSkippingIndexParams(parameters); + return indexBuilder; + } +} diff --git a/src/EFCore.ClickHouse/Extensions/ClickHouseIndexExtensions.cs b/src/EFCore.ClickHouse/Extensions/ClickHouseIndexExtensions.cs new file mode 100644 index 0000000..855a607 --- /dev/null +++ b/src/EFCore.ClickHouse/Extensions/ClickHouseIndexExtensions.cs @@ -0,0 +1,31 @@ +using ClickHouse.EntityFrameworkCore.Metadata.Internal; +using Microsoft.EntityFrameworkCore.Metadata; + +namespace ClickHouse.EntityFrameworkCore.Extensions; + +public static class ClickHouseIndexExtensions +{ + // Skipping index type (minmax, set, bloom_filter, etc.) + + public static string? GetSkippingIndexType(this IReadOnlyIndex index) + => (string?)index[ClickHouseAnnotationNames.SkippingIndexType]; + + public static void SetSkippingIndexType(this IMutableIndex index, string? type) + => index.SetOrRemoveAnnotation(ClickHouseAnnotationNames.SkippingIndexType, type); + + // Granularity + + public static int? GetGranularity(this IReadOnlyIndex index) + => (int?)index[ClickHouseAnnotationNames.SkippingIndexGranularity]; + + public static void SetGranularity(this IMutableIndex index, int? granularity) + => index.SetOrRemoveAnnotation(ClickHouseAnnotationNames.SkippingIndexGranularity, granularity); + + // Skipping index params (e.g., "100" for set(100), "0.01" for bloom_filter(0.01)) + + public static string? GetSkippingIndexParams(this IReadOnlyIndex index) + => (string?)index[ClickHouseAnnotationNames.SkippingIndexParams]; + + public static void SetSkippingIndexParams(this IMutableIndex index, string? parameters) + => index.SetOrRemoveAnnotation(ClickHouseAnnotationNames.SkippingIndexParams, parameters); +} diff --git a/src/EFCore.ClickHouse/Extensions/ClickHousePropertyBuilderExtensions.cs b/src/EFCore.ClickHouse/Extensions/ClickHousePropertyBuilderExtensions.cs new file mode 100644 index 0000000..7e62e51 --- /dev/null +++ b/src/EFCore.ClickHouse/Extensions/ClickHousePropertyBuilderExtensions.cs @@ -0,0 +1,46 @@ +using ClickHouse.EntityFrameworkCore.Metadata.Internal; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +namespace ClickHouse.EntityFrameworkCore.Extensions; + +public static class ClickHousePropertyBuilderExtensions +{ + public static PropertyBuilder HasCodec(this PropertyBuilder propertyBuilder, string codec) + { + propertyBuilder.Metadata.SetCodec(codec); + return propertyBuilder; + } + + public static PropertyBuilder HasCodec( + this PropertyBuilder propertyBuilder, string codec) + { + propertyBuilder.Metadata.SetCodec(codec); + return propertyBuilder; + } + + public static PropertyBuilder HasColumnTtl(this PropertyBuilder propertyBuilder, string ttlExpression) + { + propertyBuilder.Metadata.SetColumnTtl(ttlExpression); + return propertyBuilder; + } + + public static PropertyBuilder HasColumnTtl( + this PropertyBuilder propertyBuilder, string ttlExpression) + { + propertyBuilder.Metadata.SetColumnTtl(ttlExpression); + return propertyBuilder; + } + + public static PropertyBuilder HasColumnComment(this PropertyBuilder propertyBuilder, string comment) + { + propertyBuilder.Metadata.SetColumnComment(comment); + return propertyBuilder; + } + + public static PropertyBuilder HasColumnComment( + this PropertyBuilder propertyBuilder, string comment) + { + propertyBuilder.Metadata.SetColumnComment(comment); + return propertyBuilder; + } +} diff --git a/src/EFCore.ClickHouse/Extensions/ClickHousePropertyExtensions.cs b/src/EFCore.ClickHouse/Extensions/ClickHousePropertyExtensions.cs new file mode 100644 index 0000000..eae4efc --- /dev/null +++ b/src/EFCore.ClickHouse/Extensions/ClickHousePropertyExtensions.cs @@ -0,0 +1,31 @@ +using ClickHouse.EntityFrameworkCore.Metadata.Internal; +using Microsoft.EntityFrameworkCore.Metadata; + +namespace ClickHouse.EntityFrameworkCore.Extensions; + +public static class ClickHousePropertyExtensions +{ + // Codec + + public static string? GetCodec(this IReadOnlyProperty property) + => (string?)property[ClickHouseAnnotationNames.ColumnCodec]; + + public static void SetCodec(this IMutableProperty property, string? codec) + => property.SetOrRemoveAnnotation(ClickHouseAnnotationNames.ColumnCodec, codec); + + // Column TTL + + public static string? GetColumnTtl(this IReadOnlyProperty property) + => (string?)property[ClickHouseAnnotationNames.ColumnTtl]; + + public static void SetColumnTtl(this IMutableProperty property, string? ttlExpression) + => property.SetOrRemoveAnnotation(ClickHouseAnnotationNames.ColumnTtl, ttlExpression); + + // Column Comment + + public static string? GetColumnComment(this IReadOnlyProperty property) + => (string?)property[ClickHouseAnnotationNames.ColumnComment]; + + public static void SetColumnComment(this IMutableProperty property, string? comment) + => property.SetOrRemoveAnnotation(ClickHouseAnnotationNames.ColumnComment, comment); +} diff --git a/src/EFCore.ClickHouse/Extensions/ClickHouseServiceCollectionExtensions.cs b/src/EFCore.ClickHouse/Extensions/ClickHouseServiceCollectionExtensions.cs index 4d9528c..e0f4861 100644 --- a/src/EFCore.ClickHouse/Extensions/ClickHouseServiceCollectionExtensions.cs +++ b/src/EFCore.ClickHouse/Extensions/ClickHouseServiceCollectionExtensions.cs @@ -7,8 +7,11 @@ using ClickHouse.EntityFrameworkCore.Query.ExpressionTranslators.Internal; using ClickHouse.EntityFrameworkCore.Query.Internal; using ClickHouse.EntityFrameworkCore.Storage.Internal; +using ClickHouse.EntityFrameworkCore.Migrations; +using ClickHouse.EntityFrameworkCore.Migrations.Internal; using ClickHouse.EntityFrameworkCore.Update.Internal; using Microsoft.EntityFrameworkCore.Diagnostics; +using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Metadata.Conventions.Infrastructure; @@ -35,6 +38,9 @@ public static IServiceCollection AddEntityFrameworkClickHouse(this IServiceColle .TryAdd(p => p.GetRequiredService()) .TryAdd() .TryAdd() + .TryAdd() + .TryAdd() + .TryAdd() .TryAdd() .TryAdd() .TryAdd() diff --git a/src/EFCore.ClickHouse/Infrastructure/Internal/ClickHouseModelValidator.cs b/src/EFCore.ClickHouse/Infrastructure/Internal/ClickHouseModelValidator.cs index cdea2d2..a051375 100644 --- a/src/EFCore.ClickHouse/Infrastructure/Internal/ClickHouseModelValidator.cs +++ b/src/EFCore.ClickHouse/Infrastructure/Internal/ClickHouseModelValidator.cs @@ -1,3 +1,5 @@ +using ClickHouse.EntityFrameworkCore.Extensions; +using ClickHouse.EntityFrameworkCore.Metadata.Internal; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Diagnostics; using Microsoft.EntityFrameworkCore.Infrastructure; @@ -20,6 +22,7 @@ public override void Validate(IModel model, IDiagnosticsLogger logger) + { + foreach (var entityType in model.GetEntityTypes()) + { + if (entityType.IsOwned() || entityType.GetTableName() is null) + continue; + + var engine = entityType.GetEngine(); + if (engine is null) + continue; + + // Log engines should not have ORDER BY/PARTITION BY + if (engine is ClickHouseAnnotationNames.TinyLog + or ClickHouseAnnotationNames.StripeLog + or ClickHouseAnnotationNames.Log + or ClickHouseAnnotationNames.Memory) + { + if (entityType.GetOrderBy() is not null) + { + logger.Logger.Log(LogLevel.Warning, + "Entity type '{EntityType}' uses the '{Engine}' engine which does not support ORDER BY.", + entityType.DisplayName(), engine); + } + } + + // Validate CollapsingMergeTree sign column exists + if (engine is ClickHouseAnnotationNames.CollapsingMergeTree) + { + var sign = entityType.GetCollapsingMergeTreeSign(); + if (sign is not null && !HasPropertyWithColumn(entityType, sign)) + { + logger.Logger.Log(LogLevel.Warning, + "Entity type '{EntityType}' uses CollapsingMergeTree with sign column '{Sign}' " + + "which does not match any property.", + entityType.DisplayName(), sign); + } + } + + // Validate ReplacingMergeTree version/isDeleted columns exist + if (engine is ClickHouseAnnotationNames.ReplacingMergeTree) + { + var version = entityType.GetReplacingMergeTreeVersion(); + if (version is not null && !HasPropertyWithColumn(entityType, version)) + { + logger.Logger.Log(LogLevel.Warning, + "Entity type '{EntityType}' uses ReplacingMergeTree with version column '{Version}' " + + "which does not match any property.", + entityType.DisplayName(), version); + } + } + } + } + + private static bool HasPropertyWithColumn(IEntityType entityType, string columnName) + => entityType.GetProperties().Any(p => + string.Equals(p.GetColumnName(), columnName, StringComparison.Ordinal) + || string.Equals(p.Name, columnName, StringComparison.Ordinal)); } diff --git a/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseAggregatingMergeTreeEngineBuilder.cs b/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseAggregatingMergeTreeEngineBuilder.cs new file mode 100644 index 0000000..b0c0b9e --- /dev/null +++ b/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseAggregatingMergeTreeEngineBuilder.cs @@ -0,0 +1,48 @@ +using ClickHouse.EntityFrameworkCore.Metadata.Internal; +using Microsoft.EntityFrameworkCore.Metadata; + +namespace ClickHouse.EntityFrameworkCore.Metadata.Builders; + +public class ClickHouseAggregatingMergeTreeEngineBuilder : ClickHouseEngineBuilder +{ + public ClickHouseAggregatingMergeTreeEngineBuilder(IMutableEntityType entityType) + : base(entityType, ClickHouseAnnotationNames.AggregatingMergeTree) + { + } + + public new ClickHouseAggregatingMergeTreeEngineBuilder WithOrderBy(params string[] columns) + { + base.WithOrderBy(columns); + return this; + } + + public new ClickHouseAggregatingMergeTreeEngineBuilder WithPartitionBy(params string[] columns) + { + base.WithPartitionBy(columns); + return this; + } + + public new ClickHouseAggregatingMergeTreeEngineBuilder WithPrimaryKey(params string[] columns) + { + base.WithPrimaryKey(columns); + return this; + } + + public new ClickHouseAggregatingMergeTreeEngineBuilder WithSampleBy(params string[] columns) + { + base.WithSampleBy(columns); + return this; + } + + public new ClickHouseAggregatingMergeTreeEngineBuilder WithTtl(string ttlExpression) + { + base.WithTtl(ttlExpression); + return this; + } + + public new ClickHouseAggregatingMergeTreeEngineBuilder WithSetting(string key, string value) + { + base.WithSetting(key, value); + return this; + } +} diff --git a/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseCollapsingMergeTreeEngineBuilder.cs b/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseCollapsingMergeTreeEngineBuilder.cs new file mode 100644 index 0000000..ec06535 --- /dev/null +++ b/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseCollapsingMergeTreeEngineBuilder.cs @@ -0,0 +1,50 @@ +using ClickHouse.EntityFrameworkCore.Extensions; +using ClickHouse.EntityFrameworkCore.Metadata.Internal; +using Microsoft.EntityFrameworkCore.Metadata; + +namespace ClickHouse.EntityFrameworkCore.Metadata.Builders; + +public class ClickHouseCollapsingMergeTreeEngineBuilder : ClickHouseEngineBuilder +{ + public ClickHouseCollapsingMergeTreeEngineBuilder(IMutableEntityType entityType, string sign) + : base(entityType, ClickHouseAnnotationNames.CollapsingMergeTree) + { + entityType.SetCollapsingMergeTreeSign(sign); + } + + public new ClickHouseCollapsingMergeTreeEngineBuilder WithOrderBy(params string[] columns) + { + base.WithOrderBy(columns); + return this; + } + + public new ClickHouseCollapsingMergeTreeEngineBuilder WithPartitionBy(params string[] columns) + { + base.WithPartitionBy(columns); + return this; + } + + public new ClickHouseCollapsingMergeTreeEngineBuilder WithPrimaryKey(params string[] columns) + { + base.WithPrimaryKey(columns); + return this; + } + + public new ClickHouseCollapsingMergeTreeEngineBuilder WithSampleBy(params string[] columns) + { + base.WithSampleBy(columns); + return this; + } + + public new ClickHouseCollapsingMergeTreeEngineBuilder WithTtl(string ttlExpression) + { + base.WithTtl(ttlExpression); + return this; + } + + public new ClickHouseCollapsingMergeTreeEngineBuilder WithSetting(string key, string value) + { + base.WithSetting(key, value); + return this; + } +} diff --git a/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseEngineBuilder.cs b/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseEngineBuilder.cs new file mode 100644 index 0000000..5b57490 --- /dev/null +++ b/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseEngineBuilder.cs @@ -0,0 +1,61 @@ +using ClickHouse.EntityFrameworkCore.Extensions; +using Microsoft.EntityFrameworkCore.Metadata; + +namespace ClickHouse.EntityFrameworkCore.Metadata.Builders; + +public abstract class ClickHouseEngineBuilder +{ + protected IMutableEntityType EntityType { get; } + + protected ClickHouseEngineBuilder(IMutableEntityType entityType, string engineName) + { + ArgumentNullException.ThrowIfNull(entityType); + ArgumentException.ThrowIfNullOrWhiteSpace(engineName); + + EntityType = entityType; + entityType.SetEngine(engineName); + } + + public ClickHouseEngineBuilder WithOrderBy(params string[] columns) + { + ArgumentNullException.ThrowIfNull(columns); + EntityType.SetOrderBy(columns); + return this; + } + + public ClickHouseEngineBuilder WithPartitionBy(params string[] columns) + { + ArgumentNullException.ThrowIfNull(columns); + EntityType.SetPartitionBy(columns); + return this; + } + + public ClickHouseEngineBuilder WithPrimaryKey(params string[] columns) + { + ArgumentNullException.ThrowIfNull(columns); + EntityType.SetClickHousePrimaryKey(columns); + return this; + } + + public ClickHouseEngineBuilder WithSampleBy(params string[] columns) + { + ArgumentNullException.ThrowIfNull(columns); + EntityType.SetSampleBy(columns); + return this; + } + + public ClickHouseEngineBuilder WithTtl(string ttlExpression) + { + ArgumentException.ThrowIfNullOrWhiteSpace(ttlExpression); + EntityType.SetTtl(ttlExpression); + return this; + } + + public ClickHouseEngineBuilder WithSetting(string key, string value) + { + ArgumentException.ThrowIfNullOrWhiteSpace(key); + ArgumentNullException.ThrowIfNull(value); + EntityType.SetSetting(key, value); + return this; + } +} diff --git a/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseGraphiteMergeTreeEngineBuilder.cs b/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseGraphiteMergeTreeEngineBuilder.cs new file mode 100644 index 0000000..3b37778 --- /dev/null +++ b/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseGraphiteMergeTreeEngineBuilder.cs @@ -0,0 +1,50 @@ +using ClickHouse.EntityFrameworkCore.Extensions; +using ClickHouse.EntityFrameworkCore.Metadata.Internal; +using Microsoft.EntityFrameworkCore.Metadata; + +namespace ClickHouse.EntityFrameworkCore.Metadata.Builders; + +public class ClickHouseGraphiteMergeTreeEngineBuilder : ClickHouseEngineBuilder +{ + public ClickHouseGraphiteMergeTreeEngineBuilder(IMutableEntityType entityType, string configSection) + : base(entityType, ClickHouseAnnotationNames.GraphiteMergeTree) + { + entityType.SetGraphiteMergeTreeConfigSection(configSection); + } + + public new ClickHouseGraphiteMergeTreeEngineBuilder WithOrderBy(params string[] columns) + { + base.WithOrderBy(columns); + return this; + } + + public new ClickHouseGraphiteMergeTreeEngineBuilder WithPartitionBy(params string[] columns) + { + base.WithPartitionBy(columns); + return this; + } + + public new ClickHouseGraphiteMergeTreeEngineBuilder WithPrimaryKey(params string[] columns) + { + base.WithPrimaryKey(columns); + return this; + } + + public new ClickHouseGraphiteMergeTreeEngineBuilder WithSampleBy(params string[] columns) + { + base.WithSampleBy(columns); + return this; + } + + public new ClickHouseGraphiteMergeTreeEngineBuilder WithTtl(string ttlExpression) + { + base.WithTtl(ttlExpression); + return this; + } + + public new ClickHouseGraphiteMergeTreeEngineBuilder WithSetting(string key, string value) + { + base.WithSetting(key, value); + return this; + } +} diff --git a/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseMergeTreeEngineBuilder.cs b/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseMergeTreeEngineBuilder.cs new file mode 100644 index 0000000..160b965 --- /dev/null +++ b/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseMergeTreeEngineBuilder.cs @@ -0,0 +1,48 @@ +using ClickHouse.EntityFrameworkCore.Metadata.Internal; +using Microsoft.EntityFrameworkCore.Metadata; + +namespace ClickHouse.EntityFrameworkCore.Metadata.Builders; + +public class ClickHouseMergeTreeEngineBuilder : ClickHouseEngineBuilder +{ + public ClickHouseMergeTreeEngineBuilder(IMutableEntityType entityType) + : base(entityType, ClickHouseAnnotationNames.MergeTree) + { + } + + public new ClickHouseMergeTreeEngineBuilder WithOrderBy(params string[] columns) + { + base.WithOrderBy(columns); + return this; + } + + public new ClickHouseMergeTreeEngineBuilder WithPartitionBy(params string[] columns) + { + base.WithPartitionBy(columns); + return this; + } + + public new ClickHouseMergeTreeEngineBuilder WithPrimaryKey(params string[] columns) + { + base.WithPrimaryKey(columns); + return this; + } + + public new ClickHouseMergeTreeEngineBuilder WithSampleBy(params string[] columns) + { + base.WithSampleBy(columns); + return this; + } + + public new ClickHouseMergeTreeEngineBuilder WithTtl(string ttlExpression) + { + base.WithTtl(ttlExpression); + return this; + } + + public new ClickHouseMergeTreeEngineBuilder WithSetting(string key, string value) + { + base.WithSetting(key, value); + return this; + } +} diff --git a/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseReplacingMergeTreeEngineBuilder.cs b/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseReplacingMergeTreeEngineBuilder.cs new file mode 100644 index 0000000..5ca8fb9 --- /dev/null +++ b/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseReplacingMergeTreeEngineBuilder.cs @@ -0,0 +1,54 @@ +using ClickHouse.EntityFrameworkCore.Extensions; +using ClickHouse.EntityFrameworkCore.Metadata.Internal; +using Microsoft.EntityFrameworkCore.Metadata; + +namespace ClickHouse.EntityFrameworkCore.Metadata.Builders; + +public class ClickHouseReplacingMergeTreeEngineBuilder : ClickHouseEngineBuilder +{ + public ClickHouseReplacingMergeTreeEngineBuilder( + IMutableEntityType entityType, string? version = null, string? isDeleted = null) + : base(entityType, ClickHouseAnnotationNames.ReplacingMergeTree) + { + if (version is not null) + entityType.SetReplacingMergeTreeVersion(version); + if (isDeleted is not null) + entityType.SetReplacingMergeTreeIsDeleted(isDeleted); + } + + public new ClickHouseReplacingMergeTreeEngineBuilder WithOrderBy(params string[] columns) + { + base.WithOrderBy(columns); + return this; + } + + public new ClickHouseReplacingMergeTreeEngineBuilder WithPartitionBy(params string[] columns) + { + base.WithPartitionBy(columns); + return this; + } + + public new ClickHouseReplacingMergeTreeEngineBuilder WithPrimaryKey(params string[] columns) + { + base.WithPrimaryKey(columns); + return this; + } + + public new ClickHouseReplacingMergeTreeEngineBuilder WithSampleBy(params string[] columns) + { + base.WithSampleBy(columns); + return this; + } + + public new ClickHouseReplacingMergeTreeEngineBuilder WithTtl(string ttlExpression) + { + base.WithTtl(ttlExpression); + return this; + } + + public new ClickHouseReplacingMergeTreeEngineBuilder WithSetting(string key, string value) + { + base.WithSetting(key, value); + return this; + } +} diff --git a/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseSimpleEngineBuilder.cs b/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseSimpleEngineBuilder.cs new file mode 100644 index 0000000..f1eb0a9 --- /dev/null +++ b/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseSimpleEngineBuilder.cs @@ -0,0 +1,40 @@ +using ClickHouse.EntityFrameworkCore.Extensions; +using Microsoft.EntityFrameworkCore.Metadata; + +namespace ClickHouse.EntityFrameworkCore.Metadata.Builders; + +/// +/// Engine builder for simple engines (TinyLog, StripeLog, Log, Memory) that do not support +/// ORDER BY, PARTITION BY, PRIMARY KEY, SAMPLE BY, TTL, or SETTINGS. +/// +public class ClickHouseSimpleEngineBuilder : ClickHouseEngineBuilder +{ + public ClickHouseSimpleEngineBuilder(IMutableEntityType entityType, string engineName) + : base(entityType, engineName) + { + } + + public new ClickHouseSimpleEngineBuilder WithOrderBy(params string[] columns) + => throw new InvalidOperationException( + $"The '{EntityType.GetEngine()}' engine does not support ORDER BY."); + + public new ClickHouseSimpleEngineBuilder WithPartitionBy(params string[] columns) + => throw new InvalidOperationException( + $"The '{EntityType.GetEngine()}' engine does not support PARTITION BY."); + + public new ClickHouseSimpleEngineBuilder WithPrimaryKey(params string[] columns) + => throw new InvalidOperationException( + $"The '{EntityType.GetEngine()}' engine does not support PRIMARY KEY."); + + public new ClickHouseSimpleEngineBuilder WithSampleBy(params string[] columns) + => throw new InvalidOperationException( + $"The '{EntityType.GetEngine()}' engine does not support SAMPLE BY."); + + public new ClickHouseSimpleEngineBuilder WithTtl(string ttlExpression) + => throw new InvalidOperationException( + $"The '{EntityType.GetEngine()}' engine does not support TTL."); + + public new ClickHouseSimpleEngineBuilder WithSetting(string key, string value) + => throw new InvalidOperationException( + $"The '{EntityType.GetEngine()}' engine does not support SETTINGS."); +} diff --git a/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseSummingMergeTreeEngineBuilder.cs b/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseSummingMergeTreeEngineBuilder.cs new file mode 100644 index 0000000..cbe3d42 --- /dev/null +++ b/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseSummingMergeTreeEngineBuilder.cs @@ -0,0 +1,51 @@ +using ClickHouse.EntityFrameworkCore.Extensions; +using ClickHouse.EntityFrameworkCore.Metadata.Internal; +using Microsoft.EntityFrameworkCore.Metadata; + +namespace ClickHouse.EntityFrameworkCore.Metadata.Builders; + +public class ClickHouseSummingMergeTreeEngineBuilder : ClickHouseEngineBuilder +{ + public ClickHouseSummingMergeTreeEngineBuilder(IMutableEntityType entityType, params string[] columns) + : base(entityType, ClickHouseAnnotationNames.SummingMergeTree) + { + if (columns.Length > 0) + entityType.SetSummingMergeTreeColumns(columns); + } + + public new ClickHouseSummingMergeTreeEngineBuilder WithOrderBy(params string[] columns) + { + base.WithOrderBy(columns); + return this; + } + + public new ClickHouseSummingMergeTreeEngineBuilder WithPartitionBy(params string[] columns) + { + base.WithPartitionBy(columns); + return this; + } + + public new ClickHouseSummingMergeTreeEngineBuilder WithPrimaryKey(params string[] columns) + { + base.WithPrimaryKey(columns); + return this; + } + + public new ClickHouseSummingMergeTreeEngineBuilder WithSampleBy(params string[] columns) + { + base.WithSampleBy(columns); + return this; + } + + public new ClickHouseSummingMergeTreeEngineBuilder WithTtl(string ttlExpression) + { + base.WithTtl(ttlExpression); + return this; + } + + public new ClickHouseSummingMergeTreeEngineBuilder WithSetting(string key, string value) + { + base.WithSetting(key, value); + return this; + } +} diff --git a/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseVersionedCollapsingMergeTreeEngineBuilder.cs b/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseVersionedCollapsingMergeTreeEngineBuilder.cs new file mode 100644 index 0000000..fd3b5c3 --- /dev/null +++ b/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseVersionedCollapsingMergeTreeEngineBuilder.cs @@ -0,0 +1,52 @@ +using ClickHouse.EntityFrameworkCore.Extensions; +using ClickHouse.EntityFrameworkCore.Metadata.Internal; +using Microsoft.EntityFrameworkCore.Metadata; + +namespace ClickHouse.EntityFrameworkCore.Metadata.Builders; + +public class ClickHouseVersionedCollapsingMergeTreeEngineBuilder : ClickHouseEngineBuilder +{ + public ClickHouseVersionedCollapsingMergeTreeEngineBuilder( + IMutableEntityType entityType, string sign, string version) + : base(entityType, ClickHouseAnnotationNames.VersionedCollapsingMergeTree) + { + entityType.SetVersionedCollapsingMergeTreeSign(sign); + entityType.SetVersionedCollapsingMergeTreeVersion(version); + } + + public new ClickHouseVersionedCollapsingMergeTreeEngineBuilder WithOrderBy(params string[] columns) + { + base.WithOrderBy(columns); + return this; + } + + public new ClickHouseVersionedCollapsingMergeTreeEngineBuilder WithPartitionBy(params string[] columns) + { + base.WithPartitionBy(columns); + return this; + } + + public new ClickHouseVersionedCollapsingMergeTreeEngineBuilder WithPrimaryKey(params string[] columns) + { + base.WithPrimaryKey(columns); + return this; + } + + public new ClickHouseVersionedCollapsingMergeTreeEngineBuilder WithSampleBy(params string[] columns) + { + base.WithSampleBy(columns); + return this; + } + + public new ClickHouseVersionedCollapsingMergeTreeEngineBuilder WithTtl(string ttlExpression) + { + base.WithTtl(ttlExpression); + return this; + } + + public new ClickHouseVersionedCollapsingMergeTreeEngineBuilder WithSetting(string key, string value) + { + base.WithSetting(key, value); + return this; + } +} diff --git a/src/EFCore.ClickHouse/Metadata/Conventions/ClickHouseConventionSetBuilder.cs b/src/EFCore.ClickHouse/Metadata/Conventions/ClickHouseConventionSetBuilder.cs index f6bbf75..af8ec2a 100644 --- a/src/EFCore.ClickHouse/Metadata/Conventions/ClickHouseConventionSetBuilder.cs +++ b/src/EFCore.ClickHouse/Metadata/Conventions/ClickHouseConventionSetBuilder.cs @@ -15,8 +15,32 @@ public ClickHouseConventionSetBuilder( public override ConventionSet CreateConventionSet() { var conventionSet = base.CreateConventionSet(); - // ClickHouse doesn't support auto-increment. - // Remove ValueGenerationConvention if needed, or handle in model validator. + + // ClickHouse doesn't support foreign keys — remove all FK-related conventions + // to prevent EF from creating implicit indexes for FK columns. + RemoveForeignKeyIndexConvention(conventionSet.EntityTypeBaseTypeChangedConventions); + conventionSet.ForeignKeyAddedConventions.Clear(); + conventionSet.ForeignKeyAnnotationChangedConventions.Clear(); + conventionSet.ForeignKeyDependentRequirednessChangedConventions.Clear(); + conventionSet.ForeignKeyOwnershipChangedConventions.Clear(); + conventionSet.ForeignKeyPrincipalEndChangedConventions.Clear(); + conventionSet.ForeignKeyPropertiesChangedConventions.Clear(); + conventionSet.ForeignKeyRemovedConventions.Clear(); + conventionSet.ForeignKeyRequirednessChangedConventions.Clear(); + conventionSet.ForeignKeyUniquenessChangedConventions.Clear(); + conventionSet.SkipNavigationForeignKeyChangedConventions.Clear(); + + conventionSet.ModelFinalizingConventions.Add(new ClickHouseDefaultEngineConvention()); + return conventionSet; } + + private static void RemoveForeignKeyIndexConvention(IList conventions) + { + for (var i = conventions.Count - 1; i >= 0; i--) + { + if (conventions[i] is ForeignKeyIndexConvention) + conventions.RemoveAt(i); + } + } } diff --git a/src/EFCore.ClickHouse/Metadata/Conventions/ClickHouseDefaultEngineConvention.cs b/src/EFCore.ClickHouse/Metadata/Conventions/ClickHouseDefaultEngineConvention.cs new file mode 100644 index 0000000..4a12c29 --- /dev/null +++ b/src/EFCore.ClickHouse/Metadata/Conventions/ClickHouseDefaultEngineConvention.cs @@ -0,0 +1,60 @@ +using ClickHouse.EntityFrameworkCore.Extensions; +using ClickHouse.EntityFrameworkCore.Metadata.Internal; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Metadata.Builders; +using Microsoft.EntityFrameworkCore.Metadata.Conventions; + +namespace ClickHouse.EntityFrameworkCore.Metadata.Conventions; + +/// +/// Sets MergeTree as the default engine for entity types that don't have an explicit engine configured. +/// Uses the EF primary key columns as ORDER BY when no explicit ORDER BY is set. +/// +public class ClickHouseDefaultEngineConvention : IModelFinalizingConvention +{ + public void ProcessModelFinalizing(IConventionModelBuilder modelBuilder, IConventionContext context) + { + foreach (var entityType in modelBuilder.Metadata.GetEntityTypes()) + { + // Skip owned types and those without a table mapping + if (entityType.IsOwned() || entityType.GetTableName() is null) + continue; + + var mutableEntityType = (IMutableEntityType)entityType; + + // Set default engine to MergeTree if none configured + if (entityType.GetEngine() is null) + { + mutableEntityType.SetEngine(ClickHouseAnnotationNames.MergeTree); + } + + // Set ORDER BY from primary key if none configured and engine is MergeTree-family + var engine = entityType.GetEngine(); + if (entityType.GetOrderBy() is null && IsMergeTreeFamily(engine)) + { + var primaryKey = entityType.FindPrimaryKey(); + if (primaryKey is not null) + { + var columns = primaryKey.Properties + .Select(p => p.GetColumnName() ?? p.Name) + .ToArray(); + mutableEntityType.SetOrderBy(columns); + } + else + { + mutableEntityType.SetOrderBy(["tuple()"]); + } + } + } + } + + private static bool IsMergeTreeFamily(string? engine) + => engine is ClickHouseAnnotationNames.MergeTree + or ClickHouseAnnotationNames.ReplacingMergeTree + or ClickHouseAnnotationNames.SummingMergeTree + or ClickHouseAnnotationNames.AggregatingMergeTree + or ClickHouseAnnotationNames.CollapsingMergeTree + or ClickHouseAnnotationNames.VersionedCollapsingMergeTree + or ClickHouseAnnotationNames.GraphiteMergeTree; +} diff --git a/src/EFCore.ClickHouse/Metadata/Internal/ClickHouseAnnotationNames.cs b/src/EFCore.ClickHouse/Metadata/Internal/ClickHouseAnnotationNames.cs index 8a9eb97..b72723d 100644 --- a/src/EFCore.ClickHouse/Metadata/Internal/ClickHouseAnnotationNames.cs +++ b/src/EFCore.ClickHouse/Metadata/Internal/ClickHouseAnnotationNames.cs @@ -3,8 +3,47 @@ namespace ClickHouse.EntityFrameworkCore.Metadata.Internal; public static class ClickHouseAnnotationNames { public const string Prefix = "ClickHouse:"; + + // Table-level engine configuration public const string Engine = Prefix + "Engine"; public const string OrderBy = Prefix + "OrderBy"; public const string PartitionBy = Prefix + "PartitionBy"; public const string PrimaryKey = Prefix + "PrimaryKey"; + public const string SampleBy = Prefix + "SampleBy"; + public const string Ttl = Prefix + "Ttl"; + + // Engine-specific parameters + public const string ReplacingMergeTreeVersion = Prefix + "ReplacingMergeTree:Version"; + public const string ReplacingMergeTreeIsDeleted = Prefix + "ReplacingMergeTree:IsDeleted"; + public const string SummingMergeTreeColumns = Prefix + "SummingMergeTree:Columns"; + public const string CollapsingMergeTreeSign = Prefix + "CollapsingMergeTree:Sign"; + public const string VersionedCollapsingMergeTreeSign = Prefix + "VersionedCollapsingMergeTree:Sign"; + public const string VersionedCollapsingMergeTreeVersion = Prefix + "VersionedCollapsingMergeTree:Version"; + public const string GraphiteMergeTreeConfigSection = Prefix + "GraphiteMergeTree:ConfigSection"; + + // Settings (prefix-based key-value storage) + public const string SettingPrefix = Prefix + "Setting:"; + + // Column-level annotations + public const string ColumnCodec = Prefix + "ColumnCodec"; + public const string ColumnTtl = Prefix + "ColumnTtl"; + public const string ColumnComment = Prefix + "ColumnComment"; + + // Data-skipping index annotations + public const string SkippingIndexType = Prefix + "SkippingIndex:Type"; + public const string SkippingIndexGranularity = Prefix + "SkippingIndex:Granularity"; + public const string SkippingIndexParams = Prefix + "SkippingIndex:Params"; + + // Engine name constants + public const string MergeTree = "MergeTree"; + public const string ReplacingMergeTree = "ReplacingMergeTree"; + public const string SummingMergeTree = "SummingMergeTree"; + public const string AggregatingMergeTree = "AggregatingMergeTree"; + public const string CollapsingMergeTree = "CollapsingMergeTree"; + public const string VersionedCollapsingMergeTree = "VersionedCollapsingMergeTree"; + public const string GraphiteMergeTree = "GraphiteMergeTree"; + public const string TinyLog = "TinyLog"; + public const string StripeLog = "StripeLog"; + public const string Log = "Log"; + public const string Memory = "Memory"; } diff --git a/src/EFCore.ClickHouse/Metadata/Internal/ClickHouseAnnotationProvider.cs b/src/EFCore.ClickHouse/Metadata/Internal/ClickHouseAnnotationProvider.cs index a86a7ca..670edcb 100644 --- a/src/EFCore.ClickHouse/Metadata/Internal/ClickHouseAnnotationProvider.cs +++ b/src/EFCore.ClickHouse/Metadata/Internal/ClickHouseAnnotationProvider.cs @@ -1,3 +1,4 @@ +using ClickHouse.EntityFrameworkCore.Extensions; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Metadata; @@ -9,4 +10,46 @@ public ClickHouseAnnotationProvider(RelationalAnnotationProviderDependencies dep : base(dependencies) { } + + public override IEnumerable For(ITable table, bool designTime) + { + if (!designTime) + yield break; + + var entityType = (IEntityType)table.EntityTypeMappings.First().TypeBase; + + foreach (var annotation in entityType.GetAnnotations()) + { + if (annotation.Name.StartsWith(ClickHouseAnnotationNames.Prefix, StringComparison.Ordinal)) + yield return annotation; + } + } + + public override IEnumerable For(ITableIndex index, bool designTime) + { + if (!designTime) + yield break; + + var modelIndex = index.MappedIndexes.First(); + + foreach (var annotation in modelIndex.GetAnnotations()) + { + if (annotation.Name.StartsWith(ClickHouseAnnotationNames.Prefix, StringComparison.Ordinal)) + yield return annotation; + } + } + + public override IEnumerable For(IColumn column, bool designTime) + { + if (!designTime) + yield break; + + var property = column.PropertyMappings.First().Property; + + foreach (var annotation in property.GetAnnotations()) + { + if (annotation.Name.StartsWith(ClickHouseAnnotationNames.Prefix, StringComparison.Ordinal)) + yield return annotation; + } + } } diff --git a/src/EFCore.ClickHouse/Migrations/ClickHouseMigrationsSqlGenerator.cs b/src/EFCore.ClickHouse/Migrations/ClickHouseMigrationsSqlGenerator.cs new file mode 100644 index 0000000..89432dd --- /dev/null +++ b/src/EFCore.ClickHouse/Migrations/ClickHouseMigrationsSqlGenerator.cs @@ -0,0 +1,567 @@ +using ClickHouse.EntityFrameworkCore.Metadata.Internal; +using ClickHouse.EntityFrameworkCore.Migrations.Operations; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Migrations.Operations; + +namespace ClickHouse.EntityFrameworkCore.Migrations; + +public class ClickHouseMigrationsSqlGenerator : MigrationsSqlGenerator +{ + public ClickHouseMigrationsSqlGenerator(MigrationsSqlGeneratorDependencies dependencies) + : base(dependencies) + { + } + + // ClickHouse does not support transactions — suppress on all statements. + protected new void EndStatement(MigrationCommandListBuilder builder, bool suppressTransaction = true) + => base.EndStatement(builder, suppressTransaction: true); + + // Custom operation dispatch + + protected override void Generate(MigrationOperation operation, IModel? model, MigrationCommandListBuilder builder) + { + switch (operation) + { + case ClickHouseCreateDatabaseOperation createDb: + Generate(createDb, builder); + return; + case ClickHouseDropDatabaseOperation dropDb: + Generate(dropDb, builder); + return; + default: + base.Generate(operation, model, builder); + return; + } + } + + protected virtual void Generate(ClickHouseCreateDatabaseOperation operation, MigrationCommandListBuilder builder) + { + builder + .Append("CREATE DATABASE ") + .Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(operation.Name)); + EndStatement(builder, suppressTransaction: true); + } + + protected virtual void Generate(ClickHouseDropDatabaseOperation operation, MigrationCommandListBuilder builder) + { + builder + .Append("DROP DATABASE ") + .Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(operation.Name)); + EndStatement(builder, suppressTransaction: true); + } + + // CREATE TABLE with ENGINE clause + + protected override void Generate( + CreateTableOperation operation, + IModel? model, + MigrationCommandListBuilder builder, + bool terminate = true) + { + base.Generate(operation, model, builder, terminate: false); + + GenerateEngineClause(operation, builder); + + builder.AppendLine(Dependencies.SqlGenerationHelper.StatementTerminator); + EndStatement(builder); + } + + // Column definition: ClickHouse nullable wrapping, codec, TTL, comment + + protected override void ColumnDefinition( + string? schema, + string table, + string name, + ColumnOperation operation, + IModel? model, + MigrationCommandListBuilder builder) + { + if (!string.IsNullOrEmpty(operation.ComputedColumnSql)) + { + ComputedColumnDefinition(schema, table, name, operation, model, builder); + return; + } + + var columnType = operation.ColumnType ?? GetColumnType(schema, table, name, operation, model)!; + + // Wrap nullable non-array types in Nullable(T) + // Skip Json type — ClickHouse Json returns {} for NULL, not SQL NULL + if (operation.IsNullable && operation.ClrType?.IsArray != true + && !columnType.StartsWith("Nullable(", StringComparison.OrdinalIgnoreCase) + && !columnType.StartsWith("Json", StringComparison.OrdinalIgnoreCase)) + { + columnType = $"Nullable({columnType})"; + } + + builder + .Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(name)) + .Append(" ") + .Append(columnType); + + // DEFAULT + var defaultValue = operation.DefaultValueSql; + if (string.IsNullOrWhiteSpace(defaultValue) && operation.DefaultValue is not null) + { + var typeMapping = (!string.IsNullOrEmpty(operation.ColumnType) + ? Dependencies.TypeMappingSource.FindMapping(operation.DefaultValue.GetType(), operation.ColumnType) + : null) ?? Dependencies.TypeMappingSource.FindMapping(operation.DefaultValue.GetType())!; + defaultValue = typeMapping.GenerateSqlLiteral(operation.DefaultValue); + } + + if (!string.IsNullOrWhiteSpace(defaultValue)) + builder.Append(" DEFAULT ").Append(defaultValue); + + // CODEC + var codec = operation.FindAnnotation(ClickHouseAnnotationNames.ColumnCodec); + if (codec?.Value is string codecStr && !string.IsNullOrWhiteSpace(codecStr)) + builder.Append($" CODEC({codecStr})"); + + // Column TTL + var columnTtl = operation.FindAnnotation(ClickHouseAnnotationNames.ColumnTtl); + if (columnTtl?.Value is string ttlStr && !string.IsNullOrWhiteSpace(ttlStr)) + builder.Append($" TTL {ttlStr}"); + + // Column COMMENT + var comment = operation.FindAnnotation(ClickHouseAnnotationNames.ColumnComment); + if (comment?.Value is string commentStr && !string.IsNullOrWhiteSpace(commentStr)) + { + var escaped = commentStr.Replace("'", "\\'"); + builder.Append($" COMMENT '{escaped}'"); + } + } + + protected override void ComputedColumnDefinition( + string? schema, + string table, + string name, + ColumnOperation operation, + IModel? model, + MigrationCommandListBuilder builder) + { + var keyword = operation.IsStored == true ? " MATERIALIZED " : " ALIAS "; + var columnType = operation.ColumnType ?? GetColumnType(schema, table, name, operation, model)!; + + if (operation.IsNullable && operation.ClrType?.IsArray != true + && !columnType.StartsWith("Nullable(", StringComparison.OrdinalIgnoreCase)) + { + columnType = $"Nullable({columnType})"; + } + + builder + .Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(name)) + .Append(" ") + .Append(columnType) + .Append(keyword) + .Append(operation.ComputedColumnSql!); + } + + // Suppress primary key, foreign key, unique constraints — ClickHouse doesn't support them as SQL constraints + + protected override void CreateTableConstraints( + CreateTableOperation operation, + IModel? model, + MigrationCommandListBuilder builder) + { + CreateTableCheckConstraints(operation, model, builder); + } + + // ALTER TABLE operations + + protected override void Generate( + AddColumnOperation operation, + IModel? model, + MigrationCommandListBuilder builder, + bool terminate = true) + { + builder + .Append("ALTER TABLE ") + .Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(operation.Table, operation.Schema)) + .Append(" ADD COLUMN "); + + ColumnDefinition(operation, model, builder); + EndStatement(builder); + } + + protected override void Generate( + DropColumnOperation operation, + IModel? model, + MigrationCommandListBuilder builder, + bool terminate = true) + { + builder + .Append("ALTER TABLE ") + .Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(operation.Table, operation.Schema)) + .Append(" DROP COLUMN ") + .Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(operation.Name)); + + EndStatement(builder); + } + + protected override void Generate( + AlterColumnOperation operation, + IModel? model, + MigrationCommandListBuilder builder) + { + builder + .Append("ALTER TABLE ") + .Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(operation.Table, operation.Schema)) + .Append(" MODIFY COLUMN "); + + ColumnDefinition(operation.Schema, operation.Table, operation.Name, operation, model, builder); + EndStatement(builder); + } + + protected override void Generate( + RenameColumnOperation operation, + IModel? model, + MigrationCommandListBuilder builder) + { + builder + .Append("ALTER TABLE ") + .Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(operation.Table, operation.Schema)) + .Append(" RENAME COLUMN ") + .Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(operation.Name)) + .Append(" TO ") + .Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(operation.NewName)); + + EndStatement(builder); + } + + protected override void Generate( + RenameTableOperation operation, + IModel? model, + MigrationCommandListBuilder builder) + { + builder + .Append("RENAME TABLE ") + .Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(operation.Name, operation.Schema)) + .Append(" TO ") + .Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(operation.NewName!, operation.NewSchema)); + + EndStatement(builder); + } + + // ALTER TABLE — reject ClickHouse metadata changes (engine, ORDER BY, etc. are immutable) + + protected override void Generate( + AlterTableOperation operation, + IModel? model, + MigrationCommandListBuilder builder) + { + // Collect all ClickHouse annotations from old and new + var oldAnnotations = operation.OldTable.GetAnnotations() + .Where(a => a.Name.StartsWith(ClickHouseAnnotationNames.Prefix, StringComparison.Ordinal)) + .ToDictionary(a => a.Name, a => a.Value); + var newAnnotations = operation.GetAnnotations() + .Where(a => a.Name.StartsWith(ClickHouseAnnotationNames.Prefix, StringComparison.Ordinal)) + .ToDictionary(a => a.Name, a => a.Value); + + // Find any annotation that was added, removed, or changed + var allKeys = oldAnnotations.Keys.Union(newAnnotations.Keys); + foreach (var key in allKeys) + { + oldAnnotations.TryGetValue(key, out var oldVal); + newAnnotations.TryGetValue(key, out var newVal); + + if (!AnnotationValuesEqual(oldVal, newVal)) + { + var shortName = key[ClickHouseAnnotationNames.Prefix.Length..]; + throw new NotSupportedException( + $"ClickHouse does not support changing table metadata '{shortName}' via ALTER TABLE. " + + "Recreate the table instead."); + } + } + + // Delegate to base for non-ClickHouse annotation changes (e.g., comments) + base.Generate(operation, model, builder); + } + + private static bool AnnotationValuesEqual(object? a, object? b) + { + if (a is null && b is null) return true; + if (a is null || b is null) return false; + if (a is string[] arrA && b is string[] arrB) return arrA.SequenceEqual(arrB); + return a.Equals(b); + } + + // Data-skipping indices + + protected override void Generate( + CreateIndexOperation operation, + IModel? model, + MigrationCommandListBuilder builder, + bool terminate = true) + { + if (operation.IsUnique) + throw new NotSupportedException("ClickHouse does not support unique indexes."); + + var indexType = (string?)operation.FindAnnotation(ClickHouseAnnotationNames.SkippingIndexType)?.Value; + if (indexType is null) + { + // Standard index — ClickHouse doesn't support CREATE INDEX syntax + // Skip silently rather than error, since EF may generate these for PK-like indices + return; + } + + var indexParams = (string?)operation.FindAnnotation(ClickHouseAnnotationNames.SkippingIndexParams)?.Value; + var granularity = (int?)operation.FindAnnotation(ClickHouseAnnotationNames.SkippingIndexGranularity)?.Value ?? 1; + + builder + .Append("ALTER TABLE ") + .Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(operation.Table, operation.Schema)) + .Append(" ADD INDEX ") + .Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(operation.Name)) + .Append(" (") + .Append(string.Join(", ", operation.Columns.Select(c => + Dependencies.SqlGenerationHelper.DelimitIdentifier(c)))) + .Append(") TYPE ") + .Append(indexType); + + if (!string.IsNullOrWhiteSpace(indexParams)) + builder.Append($"({indexParams})"); + + builder.Append($" GRANULARITY {granularity}"); + EndStatement(builder); + } + + protected override void Generate( + DropIndexOperation operation, + IModel? model, + MigrationCommandListBuilder builder, + bool terminate = true) + { + // Only emit DROP INDEX for skipping indexes (same symmetry as CreateIndexOperation). + // Standard EF indexes are not created in ClickHouse, so dropping them is a no-op. + var indexType = (string?)operation.FindAnnotation(ClickHouseAnnotationNames.SkippingIndexType)?.Value; + if (indexType is null) + return; + + builder + .Append("ALTER TABLE ") + .Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(operation.Table!, operation.Schema)) + .Append(" DROP INDEX ") + .Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(operation.Name)); + + EndStatement(builder); + } + + // Unsupported operations + + protected override void Generate(AddForeignKeyOperation operation, IModel? model, MigrationCommandListBuilder builder, bool terminate = true) + => throw new NotSupportedException("ClickHouse does not support foreign key constraints."); + + protected override void Generate(DropForeignKeyOperation operation, IModel? model, MigrationCommandListBuilder builder, bool terminate = true) + => throw new NotSupportedException("ClickHouse does not support foreign key constraints."); + + protected override void Generate(AddUniqueConstraintOperation operation, IModel? model, MigrationCommandListBuilder builder) + => throw new NotSupportedException("ClickHouse does not support unique constraints."); + + protected override void Generate(DropUniqueConstraintOperation operation, IModel? model, MigrationCommandListBuilder builder) + => throw new NotSupportedException("ClickHouse does not support unique constraints."); + + protected override void Generate(AddPrimaryKeyOperation operation, IModel? model, MigrationCommandListBuilder builder, bool terminate = true) + { + // No-op: ClickHouse primary key is structural (ORDER BY), not a constraint + } + + protected override void Generate(DropPrimaryKeyOperation operation, IModel? model, MigrationCommandListBuilder builder, bool terminate = true) + { + // No-op: ClickHouse primary key is structural (ORDER BY), not a constraint + } + + protected override void Generate(CreateSequenceOperation operation, IModel? model, MigrationCommandListBuilder builder) + => throw new NotSupportedException("ClickHouse does not support sequences."); + + protected override void Generate(AlterSequenceOperation operation, IModel? model, MigrationCommandListBuilder builder) + => throw new NotSupportedException("ClickHouse does not support sequences."); + + protected override void Generate(DropSequenceOperation operation, IModel? model, MigrationCommandListBuilder builder) + => throw new NotSupportedException("ClickHouse does not support sequences."); + + protected override void Generate(RenameSequenceOperation operation, IModel? model, MigrationCommandListBuilder builder) + => throw new NotSupportedException("ClickHouse does not support sequences."); + + protected override void Generate(EnsureSchemaOperation operation, IModel? model, MigrationCommandListBuilder builder) + => throw new NotSupportedException("ClickHouse does not support schemas. Use databases instead."); + + // ENGINE clause generation + + private void GenerateEngineClause(CreateTableOperation operation, MigrationCommandListBuilder builder) + { + var engine = (string?)operation.FindAnnotation(ClickHouseAnnotationNames.Engine)?.Value + ?? ClickHouseAnnotationNames.MergeTree; + + builder.AppendLine(); + + // ENGINE = EngineName or ENGINE = EngineName(args) + // Simple engines (Log, TinyLog, StripeLog, Memory) use bare names without parentheses. + if (IsSimpleEngine(engine)) + { + builder.Append($"ENGINE = {engine}"); + } + else + { + builder.Append($"ENGINE = {engine}("); + GenerateEngineArgs(operation, engine, builder); + builder.Append(")"); + } + + // ORDER BY + var orderBy = (string[]?)operation.FindAnnotation(ClickHouseAnnotationNames.OrderBy)?.Value; + if (orderBy is { Length: > 0 }) + { + builder.AppendLine(); + builder.Append("ORDER BY ("); + builder.Append(string.Join(", ", orderBy.Select(QuoteColumnOrExpression))); + builder.Append(")"); + } + else if (IsMergeTreeFamily(engine)) + { + builder.AppendLine(); + builder.Append("ORDER BY tuple()"); + } + + // PARTITION BY + var partitionBy = (string[]?)operation.FindAnnotation(ClickHouseAnnotationNames.PartitionBy)?.Value; + if (partitionBy is { Length: > 0 }) + { + builder.AppendLine(); + builder.Append("PARTITION BY "); + if (partitionBy.Length == 1) + builder.Append(QuoteColumnOrExpression(partitionBy[0])); + else + { + builder.Append("("); + builder.Append(string.Join(", ", partitionBy.Select(QuoteColumnOrExpression))); + builder.Append(")"); + } + } + + // PRIMARY KEY + var primaryKey = (string[]?)operation.FindAnnotation(ClickHouseAnnotationNames.PrimaryKey)?.Value; + if (primaryKey is { Length: > 0 }) + { + builder.AppendLine(); + builder.Append("PRIMARY KEY ("); + builder.Append(string.Join(", ", primaryKey.Select(QuoteColumnOrExpression))); + builder.Append(")"); + } + + // SAMPLE BY + var sampleBy = (string[]?)operation.FindAnnotation(ClickHouseAnnotationNames.SampleBy)?.Value; + if (sampleBy is { Length: > 0 }) + { + builder.AppendLine(); + builder.Append("SAMPLE BY "); + if (sampleBy.Length == 1) + builder.Append(QuoteColumnOrExpression(sampleBy[0])); + else + { + builder.Append("("); + builder.Append(string.Join(", ", sampleBy.Select(QuoteColumnOrExpression))); + builder.Append(")"); + } + } + + // TTL + var ttl = (string?)operation.FindAnnotation(ClickHouseAnnotationNames.Ttl)?.Value; + if (!string.IsNullOrWhiteSpace(ttl)) + { + builder.AppendLine(); + builder.Append($"TTL {ttl}"); + } + + // SETTINGS + GenerateSettingsClause(operation, builder); + } + + private void GenerateEngineArgs(CreateTableOperation operation, string engine, MigrationCommandListBuilder builder) + { + switch (engine) + { + case ClickHouseAnnotationNames.ReplacingMergeTree: + var version = (string?)operation.FindAnnotation(ClickHouseAnnotationNames.ReplacingMergeTreeVersion)?.Value; + var isDeleted = (string?)operation.FindAnnotation(ClickHouseAnnotationNames.ReplacingMergeTreeIsDeleted)?.Value; + var args = new List(); + if (version is not null) + args.Add(QuoteColumnOrExpression(version)); + if (isDeleted is not null) + args.Add(QuoteColumnOrExpression(isDeleted)); + builder.Append(string.Join(", ", args)); + break; + + case ClickHouseAnnotationNames.SummingMergeTree: + var columns = (string[]?)operation.FindAnnotation(ClickHouseAnnotationNames.SummingMergeTreeColumns)?.Value; + if (columns is { Length: > 0 }) + builder.Append(string.Join(", ", columns.Select(QuoteColumnOrExpression))); + break; + + case ClickHouseAnnotationNames.CollapsingMergeTree: + var sign = (string?)operation.FindAnnotation(ClickHouseAnnotationNames.CollapsingMergeTreeSign)?.Value; + if (sign is not null) + builder.Append(QuoteColumnOrExpression(sign)); + break; + + case ClickHouseAnnotationNames.VersionedCollapsingMergeTree: + var vcSign = (string?)operation.FindAnnotation(ClickHouseAnnotationNames.VersionedCollapsingMergeTreeSign)?.Value; + var vcVersion = (string?)operation.FindAnnotation(ClickHouseAnnotationNames.VersionedCollapsingMergeTreeVersion)?.Value; + var vcArgs = new List(); + if (vcSign is not null) vcArgs.Add(QuoteColumnOrExpression(vcSign)); + if (vcVersion is not null) vcArgs.Add(QuoteColumnOrExpression(vcVersion)); + builder.Append(string.Join(", ", vcArgs)); + break; + + case ClickHouseAnnotationNames.GraphiteMergeTree: + var config = (string?)operation.FindAnnotation(ClickHouseAnnotationNames.GraphiteMergeTreeConfigSection)?.Value; + if (config is not null) + builder.Append($"'{config}'"); + break; + } + } + + private void GenerateSettingsClause(CreateTableOperation operation, MigrationCommandListBuilder builder) + { + var settings = new Dictionary(); + foreach (var annotation in operation.GetAnnotations()) + { + if (annotation.Name.StartsWith(ClickHouseAnnotationNames.SettingPrefix, StringComparison.Ordinal) + && annotation.Value is string value) + { + var key = annotation.Name[ClickHouseAnnotationNames.SettingPrefix.Length..]; + settings[key] = value; + } + } + + if (settings.Count == 0) + return; + + builder.AppendLine(); + builder.Append("SETTINGS "); + builder.Append(string.Join(", ", settings.Select(kv => $"{kv.Key} = {kv.Value}"))); + } + + private string QuoteColumnOrExpression(string columnOrExpr) + { + // If it contains '(' it's a function expression — emit verbatim + if (columnOrExpr.Contains('(')) + return columnOrExpr; + + return Dependencies.SqlGenerationHelper.DelimitIdentifier(columnOrExpr); + } + + private static bool IsMergeTreeFamily(string engine) + => engine is ClickHouseAnnotationNames.MergeTree + or ClickHouseAnnotationNames.ReplacingMergeTree + or ClickHouseAnnotationNames.SummingMergeTree + or ClickHouseAnnotationNames.AggregatingMergeTree + or ClickHouseAnnotationNames.CollapsingMergeTree + or ClickHouseAnnotationNames.VersionedCollapsingMergeTree + or ClickHouseAnnotationNames.GraphiteMergeTree; + + private static bool IsSimpleEngine(string engine) + => engine is ClickHouseAnnotationNames.TinyLog + or ClickHouseAnnotationNames.StripeLog + or ClickHouseAnnotationNames.Log + or ClickHouseAnnotationNames.Memory; +} diff --git a/src/EFCore.ClickHouse/Migrations/Internal/ClickHouseHistoryRepository.cs b/src/EFCore.ClickHouse/Migrations/Internal/ClickHouseHistoryRepository.cs new file mode 100644 index 0000000..5bb9e79 --- /dev/null +++ b/src/EFCore.ClickHouse/Migrations/Internal/ClickHouseHistoryRepository.cs @@ -0,0 +1,60 @@ +using ClickHouse.EntityFrameworkCore.Extensions; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; +using Microsoft.EntityFrameworkCore.Migrations; + +namespace ClickHouse.EntityFrameworkCore.Migrations.Internal; + +public class ClickHouseHistoryRepository : HistoryRepository +{ + public ClickHouseHistoryRepository(HistoryRepositoryDependencies dependencies) + : base(dependencies) + { + } + + protected override bool InterpretExistsResult(object? value) + => value is not null and not DBNull && Convert.ToBoolean(value); + + public override string GetCreateIfNotExistsScript() + { + var script = GetCreateScript(); + return script.Insert( + script.IndexOf("CREATE TABLE", StringComparison.Ordinal) + 12, + " IF NOT EXISTS"); + } + + public override string GetBeginIfNotExistsScript(string migrationId) + => throw new NotSupportedException( + "ClickHouse does not support conditional SQL blocks. " + + "Idempotent migration scripts (--idempotent) are not supported by this provider."); + + public override string GetBeginIfExistsScript(string migrationId) + => throw new NotSupportedException( + "ClickHouse does not support conditional SQL blocks. " + + "Idempotent migration scripts (--idempotent) are not supported by this provider."); + + public override string GetEndIfScript() + => throw new NotSupportedException( + "ClickHouse does not support conditional SQL blocks. " + + "Idempotent migration scripts (--idempotent) are not supported by this provider."); + + public override IMigrationsDatabaseLock AcquireDatabaseLock() + => new ClickHouseMigrationDatabaseLock(this); + + public override Task AcquireDatabaseLockAsync(CancellationToken cancellationToken = default) + => Task.FromResult(new ClickHouseMigrationDatabaseLock(this)); + + protected override void ConfigureTable(EntityTypeBuilder history) + { + history.Property(h => h.MigrationId).HasMaxLength(150); + history.Property(h => h.ProductVersion).HasMaxLength(32).IsRequired(); + history.ToTable(TableName, table => table + .HasMergeTreeEngine() + .WithOrderBy("MigrationId")); + } + + public override LockReleaseBehavior LockReleaseBehavior => LockReleaseBehavior.Connection; + + protected override string ExistsSql + => $"EXISTS {SqlGenerationHelper.DelimitIdentifier(TableName, TableSchema)}{SqlGenerationHelper.StatementTerminator}"; +} diff --git a/src/EFCore.ClickHouse/Migrations/Internal/ClickHouseMigrationDatabaseLock.cs b/src/EFCore.ClickHouse/Migrations/Internal/ClickHouseMigrationDatabaseLock.cs new file mode 100644 index 0000000..d42c457 --- /dev/null +++ b/src/EFCore.ClickHouse/Migrations/Internal/ClickHouseMigrationDatabaseLock.cs @@ -0,0 +1,20 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +namespace ClickHouse.EntityFrameworkCore.Migrations.Internal; + +/// +/// No-op database lock for ClickHouse (no transaction/lock support). +/// +public class ClickHouseMigrationDatabaseLock : IMigrationsDatabaseLock +{ + public ClickHouseMigrationDatabaseLock(IHistoryRepository historyRepository) + { + HistoryRepository = historyRepository; + } + + public IHistoryRepository HistoryRepository { get; } + + public void Dispose() { } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; +} diff --git a/src/EFCore.ClickHouse/Migrations/Internal/ClickHouseMigrationsAnnotationProvider.cs b/src/EFCore.ClickHouse/Migrations/Internal/ClickHouseMigrationsAnnotationProvider.cs new file mode 100644 index 0000000..8b7127c --- /dev/null +++ b/src/EFCore.ClickHouse/Migrations/Internal/ClickHouseMigrationsAnnotationProvider.cs @@ -0,0 +1,11 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +namespace ClickHouse.EntityFrameworkCore.Migrations.Internal; + +public class ClickHouseMigrationsAnnotationProvider : MigrationsAnnotationProvider +{ + public ClickHouseMigrationsAnnotationProvider(MigrationsAnnotationProviderDependencies dependencies) + : base(dependencies) + { + } +} diff --git a/src/EFCore.ClickHouse/Migrations/Operations/ClickHouseCreateDatabaseOperation.cs b/src/EFCore.ClickHouse/Migrations/Operations/ClickHouseCreateDatabaseOperation.cs new file mode 100644 index 0000000..62003e1 --- /dev/null +++ b/src/EFCore.ClickHouse/Migrations/Operations/ClickHouseCreateDatabaseOperation.cs @@ -0,0 +1,8 @@ +using Microsoft.EntityFrameworkCore.Migrations.Operations; + +namespace ClickHouse.EntityFrameworkCore.Migrations.Operations; + +public class ClickHouseCreateDatabaseOperation : MigrationOperation +{ + public required string Name { get; set; } +} diff --git a/src/EFCore.ClickHouse/Migrations/Operations/ClickHouseDropDatabaseOperation.cs b/src/EFCore.ClickHouse/Migrations/Operations/ClickHouseDropDatabaseOperation.cs new file mode 100644 index 0000000..136ee7b --- /dev/null +++ b/src/EFCore.ClickHouse/Migrations/Operations/ClickHouseDropDatabaseOperation.cs @@ -0,0 +1,8 @@ +using Microsoft.EntityFrameworkCore.Migrations.Operations; + +namespace ClickHouse.EntityFrameworkCore.Migrations.Operations; + +public class ClickHouseDropDatabaseOperation : MigrationOperation +{ + public required string Name { get; set; } +} diff --git a/test/EFCore.ClickHouse.DesignSmoke/EFCore.ClickHouse.DesignSmoke.csproj b/test/EFCore.ClickHouse.DesignSmoke/EFCore.ClickHouse.DesignSmoke.csproj new file mode 100644 index 0000000..f56283f --- /dev/null +++ b/test/EFCore.ClickHouse.DesignSmoke/EFCore.ClickHouse.DesignSmoke.csproj @@ -0,0 +1,15 @@ + + + + net10.0 + Exe + enable + enable + + + + + + + + diff --git a/test/EFCore.ClickHouse.DesignSmoke/Program.cs b/test/EFCore.ClickHouse.DesignSmoke/Program.cs new file mode 100644 index 0000000..b537287 --- /dev/null +++ b/test/EFCore.ClickHouse.DesignSmoke/Program.cs @@ -0,0 +1,2 @@ +// Minimal entry point required for dotnet-ef to discover the design-time factory. +Console.WriteLine("This project is used for dotnet-ef CLI smoke testing only."); diff --git a/test/EFCore.ClickHouse.DesignSmoke/SmokeDbContext.cs b/test/EFCore.ClickHouse.DesignSmoke/SmokeDbContext.cs new file mode 100644 index 0000000..fa9ed37 --- /dev/null +++ b/test/EFCore.ClickHouse.DesignSmoke/SmokeDbContext.cs @@ -0,0 +1,67 @@ +using ClickHouse.EntityFrameworkCore.Extensions; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Design; + +namespace EFCore.ClickHouse.DesignSmoke; + +public class SmokeDbContext : DbContext +{ + public SmokeDbContext(DbContextOptions options) : base(options) { } + + public DbSet SensorReadings => Set(); + public DbSet AuditLogs => Set(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(b => + { + b.HasKey(e => e.Id); + b.Property(e => e.Temperature).HasCodec("Delta, ZSTD"); + b.Property(e => e.Timestamp) + .HasColumnTtl("Timestamp + INTERVAL 90 DAY") + .HasColumnComment("Reading timestamp"); + b.HasIndex(e => e.Timestamp) + .HasSkippingIndexType("minmax") + .HasGranularity(4); + b.ToTable("sensor_readings", t => t + .HasReplacingMergeTreeEngine("Version") + .WithOrderBy("SensorId", "Timestamp") + .WithPartitionBy("toYYYYMM(Timestamp)") + .WithPrimaryKey("SensorId") + .WithSampleBy("SensorId") + .WithTtl("Timestamp + INTERVAL 1 YEAR") + .WithSetting("index_granularity", "4096")); + }); + + modelBuilder.Entity(b => + { + b.HasKey(e => e.Id); + b.ToTable("audit_logs", t => t.HasMemoryEngine()); + }); + } +} + +public class SensorReading +{ + public long Id { get; set; } + public string SensorId { get; set; } = string.Empty; + public DateTime Timestamp { get; set; } + public short Temperature { get; set; } + public ulong Version { get; set; } +} + +public class AuditLog +{ + public long Id { get; set; } + public string Message { get; set; } = string.Empty; +} + +public class SmokeDbContextFactory : IDesignTimeDbContextFactory +{ + public SmokeDbContext CreateDbContext(string[] args) + { + var optionsBuilder = new DbContextOptionsBuilder(); + optionsBuilder.UseClickHouse("Host=localhost;Database=smoke_test"); + return new SmokeDbContext(optionsBuilder.Options); + } +} diff --git a/test/EFCore.ClickHouse.Tests/EngineConfigurationTests.cs b/test/EFCore.ClickHouse.Tests/EngineConfigurationTests.cs new file mode 100644 index 0000000..003c37b --- /dev/null +++ b/test/EFCore.ClickHouse.Tests/EngineConfigurationTests.cs @@ -0,0 +1,368 @@ +using ClickHouse.EntityFrameworkCore.Extensions; +using ClickHouse.EntityFrameworkCore.Metadata.Internal; +using Microsoft.EntityFrameworkCore; +using Xunit; + +namespace EFCore.ClickHouse.Tests; + +public class EngineConfigurationTests +{ + [Fact] + public void HasMergeTreeEngine_sets_engine_annotation() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasMergeTreeEngine()); + }); + }); + + var entityType = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal(ClickHouseAnnotationNames.MergeTree, entityType.GetEngine()); + } + + [Fact] + public void HasReplacingMergeTreeEngine_sets_version_and_isDeleted() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasReplacingMergeTreeEngine("Version", "IsDeleted")); + }); + }); + + var entityType = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal(ClickHouseAnnotationNames.ReplacingMergeTree, entityType.GetEngine()); + Assert.Equal("Version", entityType.GetReplacingMergeTreeVersion()); + Assert.Equal("IsDeleted", entityType.GetReplacingMergeTreeIsDeleted()); + } + + [Fact] + public void HasCollapsingMergeTreeEngine_sets_sign() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasCollapsingMergeTreeEngine("Sign")); + }); + }); + + var entityType = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal(ClickHouseAnnotationNames.CollapsingMergeTree, entityType.GetEngine()); + Assert.Equal("Sign", entityType.GetCollapsingMergeTreeSign()); + } + + [Fact] + public void HasVersionedCollapsingMergeTreeEngine_sets_sign_and_version() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasVersionedCollapsingMergeTreeEngine("Sign", "Ver")); + }); + }); + + var entityType = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal(ClickHouseAnnotationNames.VersionedCollapsingMergeTree, entityType.GetEngine()); + Assert.Equal("Sign", entityType.GetVersionedCollapsingMergeTreeSign()); + Assert.Equal("Ver", entityType.GetVersionedCollapsingMergeTreeVersion()); + } + + [Fact] + public void HasSummingMergeTreeEngine_sets_columns() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasSummingMergeTreeEngine("Amount", "Count")); + }); + }); + + var entityType = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal(ClickHouseAnnotationNames.SummingMergeTree, entityType.GetEngine()); + Assert.Equal(["Amount", "Count"], entityType.GetSummingMergeTreeColumns()); + } + + [Fact] + public void HasGraphiteMergeTreeEngine_sets_config_section() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasGraphiteMergeTreeEngine("graphite_rollup")); + }); + }); + + var entityType = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal(ClickHouseAnnotationNames.GraphiteMergeTree, entityType.GetEngine()); + Assert.Equal("graphite_rollup", entityType.GetGraphiteMergeTreeConfigSection()); + } + + [Fact] + public void WithOrderBy_stores_column_array() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasMergeTreeEngine().WithOrderBy("Id", "Name")); + }); + }); + + var entityType = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal(["Id", "Name"], entityType.GetOrderBy()); + } + + [Fact] + public void WithPartitionBy_stores_expression() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasMergeTreeEngine() + .WithOrderBy("Id") + .WithPartitionBy("toYYYYMM(CreatedAt)")); + }); + }); + + var entityType = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal(["toYYYYMM(CreatedAt)"], entityType.GetPartitionBy()); + } + + [Fact] + public void WithSetting_stores_prefix_based_annotations() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasMergeTreeEngine() + .WithOrderBy("Id") + .WithSetting("index_granularity", "4096") + .WithSetting("storage_policy", "'hot_cold'")); + }); + }); + + var entityType = model.FindEntityType(typeof(TestEntity))!; + var settings = entityType.GetSettings(); + Assert.Equal(2, settings.Count); + Assert.Equal("4096", settings["index_granularity"]); + Assert.Equal("'hot_cold'", settings["storage_policy"]); + } + + [Fact] + public void WithTtl_stores_ttl_expression() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasMergeTreeEngine() + .WithOrderBy("Id") + .WithTtl("CreatedAt + INTERVAL 30 DAY")); + }); + }); + + var entityType = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal("CreatedAt + INTERVAL 30 DAY", entityType.GetTtl()); + } + + [Fact] + public void HasMemoryEngine_sets_engine() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasMemoryEngine()); + }); + }); + + var entityType = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal(ClickHouseAnnotationNames.Memory, entityType.GetEngine()); + } + + [Fact] + public void SimpleEngine_WithOrderBy_throws() + { + Assert.Throws(() => + { + BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasMemoryEngine().WithOrderBy("Id")); + }); + }); + }); + } + + [Fact] + public void Index_HasSkippingIndexType_stores_annotation() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.HasIndex(x => x.Name) + .HasSkippingIndexType("minmax") + .HasGranularity(4); + e.ToTable("test", t => t.HasMergeTreeEngine().WithOrderBy("Id")); + }); + }); + + var index = model.FindEntityType(typeof(TestEntity))!.GetIndexes().First(); + Assert.Equal("minmax", index.GetSkippingIndexType()); + Assert.Equal(4, index.GetGranularity()); + } + + [Fact] + public void Property_HasCodec_stores_annotation() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.Property(x => x.Id).HasCodec("Delta, ZSTD"); + e.ToTable("test", t => t.HasMergeTreeEngine().WithOrderBy("Id")); + }); + }); + + var property = model.FindEntityType(typeof(TestEntity))!.FindProperty("Id")!; + Assert.Equal("Delta, ZSTD", property.GetCodec()); + } + + [Fact] + public void Property_HasColumnTtl_stores_annotation() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.Property(x => x.Name).HasColumnTtl("CreatedAt + INTERVAL 1 DAY"); + e.ToTable("test", t => t.HasMergeTreeEngine().WithOrderBy("Id")); + }); + }); + + var property = model.FindEntityType(typeof(TestEntity))!.FindProperty("Name")!; + Assert.Equal("CreatedAt + INTERVAL 1 DAY", property.GetColumnTtl()); + } + + [Fact] + public void Property_HasColumnComment_stores_annotation() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.Property(x => x.Name).HasColumnComment("User's display name"); + e.ToTable("test", t => t.HasMergeTreeEngine().WithOrderBy("Id")); + }); + }); + + var property = model.FindEntityType(typeof(TestEntity))!.FindProperty("Name")!; + Assert.Equal("User's display name", property.GetColumnComment()); + } + + [Fact] + public void Default_convention_sets_MergeTree_with_PK_as_OrderBy() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test"); + }); + }); + + var entityType = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal(ClickHouseAnnotationNames.MergeTree, entityType.GetEngine()); + Assert.Equal(["Id"], entityType.GetOrderBy()); + } + + [Fact] + public void Default_convention_sets_tuple_OrderBy_when_no_PK() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasNoKey(); + e.ToTable("test"); + }); + }); + + var entityType = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal(ClickHouseAnnotationNames.MergeTree, entityType.GetEngine()); + Assert.Equal(["tuple()"], entityType.GetOrderBy()); + } + + [Fact] + public void Default_convention_does_not_override_explicit_engine() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasStripeLogEngine()); + }); + }); + + var entityType = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal(ClickHouseAnnotationNames.StripeLog, entityType.GetEngine()); + Assert.Null(entityType.GetOrderBy()); + } + + private static Microsoft.EntityFrameworkCore.Metadata.IModel BuildModel(Action configure) + { + var optionsBuilder = new DbContextOptionsBuilder() + .UseClickHouse("Host=localhost;Database=test") + .EnableServiceProviderCaching(false); + using var context = new TestDbContext(optionsBuilder.Options, configure); + return context.Model; + } + + private class TestDbContext : DbContext + { + private readonly Action _configure; + + public TestDbContext(DbContextOptions options, Action configure) + : base(options) + { + _configure = configure; + } + + protected override void OnModelCreating(ModelBuilder modelBuilder) => _configure(modelBuilder); + } + + private class TestEntity + { + public long Id { get; set; } + public string Name { get; set; } = string.Empty; + } +} diff --git a/test/EFCore.ClickHouse.Tests/EnsureCreatedTests.cs b/test/EFCore.ClickHouse.Tests/EnsureCreatedTests.cs new file mode 100644 index 0000000..1828889 --- /dev/null +++ b/test/EFCore.ClickHouse.Tests/EnsureCreatedTests.cs @@ -0,0 +1,241 @@ +using ClickHouse.EntityFrameworkCore.Extensions; +using ClickHouse.EntityFrameworkCore.Metadata.Internal; +using Microsoft.EntityFrameworkCore; +using Xunit; + +namespace EFCore.ClickHouse.Tests; + +public class EnsureCreatedTests : IAsyncLifetime +{ + private string _connectionString = default!; + + public async Task InitializeAsync() + { + _connectionString = await SharedContainer.GetConnectionStringAsync(); + } + + public Task DisposeAsync() => Task.CompletedTask; + + [Fact] + public async Task EnsureCreated_MergeTree_creates_table() + { + await using var context = CreateContext(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("mt_test", t => t + .HasMergeTreeEngine() + .WithOrderBy("Id")); + }); + }); + + await context.Database.EnsureDeletedAsync(); + await context.Database.EnsureCreatedAsync(); + + // Verify table exists and has correct engine via system.tables + var engine = await QueryScalar(context, "SELECT engine FROM system.tables WHERE name = 'mt_test'"); + Assert.Equal("MergeTree", engine); + } + + [Fact] + public async Task EnsureCreated_ReplacingMergeTree_creates_table() + { + await using var context = CreateContext(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("rmt_test", t => t + .HasReplacingMergeTreeEngine("Version") + .WithOrderBy("Id")); + }); + }); + + await context.Database.EnsureDeletedAsync(); + await context.Database.EnsureCreatedAsync(); + + var engine = await QueryScalar(context, "SELECT engine FROM system.tables WHERE name = 'rmt_test'"); + Assert.Equal("ReplacingMergeTree", engine); + } + + [Fact] + public async Task EnsureCreated_default_engine_uses_MergeTree() + { + await using var context = CreateContext(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("default_engine_test"); + }); + }); + + await context.Database.EnsureDeletedAsync(); + await context.Database.EnsureCreatedAsync(); + + var engine = await QueryScalar(context, "SELECT engine FROM system.tables WHERE name = 'default_engine_test'"); + Assert.Equal("MergeTree", engine); + } + + [Fact] + public async Task EnsureCreated_with_partitionBy_and_settings() + { + await using var context = CreateContext(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("partition_test", t => t + .HasMergeTreeEngine() + .WithOrderBy("Id") + .WithPartitionBy("toYYYYMM(Timestamp)") + .WithSetting("index_granularity", "4096")); + }); + }); + + await context.Database.EnsureDeletedAsync(); + await context.Database.EnsureCreatedAsync(); + + var engine = await QueryScalar(context, "SELECT engine FROM system.tables WHERE name = 'partition_test'"); + Assert.Equal("MergeTree", engine); + } + + [Fact] + public async Task EnsureCreated_insert_and_query_roundtrip() + { + await using var context = CreateContext(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("roundtrip_test", t => t + .HasMergeTreeEngine() + .WithOrderBy("Id")); + }); + }); + + await context.Database.EnsureDeletedAsync(); + await context.Database.EnsureCreatedAsync(); + + context.Set().Add(new SimpleEntity { Id = 1, Name = "test" }); + await context.SaveChangesAsync(); + + await using var readContext = CreateContext(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("roundtrip_test", t => t + .HasMergeTreeEngine() + .WithOrderBy("Id")); + }); + }); + + var entity = await readContext.Set().FirstAsync(e => e.Id == 1); + Assert.Equal("test", entity.Name); + } + + [Fact] + public async Task EnsureCreated_with_codec_column() + { + await using var context = CreateContext(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.Property(x => x.Id).HasCodec("Delta, ZSTD"); + e.ToTable("codec_test", t => t + .HasMergeTreeEngine() + .WithOrderBy("Id")); + }); + }); + + await context.Database.EnsureDeletedAsync(); + await context.Database.EnsureCreatedAsync(); + + // Table creation succeeds means codec was accepted + var engine = await QueryScalar(context, "SELECT engine FROM system.tables WHERE name = 'codec_test'"); + Assert.Equal("MergeTree", engine); + } + + [Fact] + public async Task EnsureDeleted_drops_database() + { + await using var context = CreateContext(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("del_test", t => t + .HasMergeTreeEngine() + .WithOrderBy("Id")); + }); + }); + + await context.Database.EnsureCreatedAsync(); + Assert.True(await context.Database.EnsureCreatedAsync() is false); // already exists + + await context.Database.EnsureDeletedAsync(); + // After delete, creating again should return true + Assert.True(await context.Database.EnsureCreatedAsync()); + } + + private TestContext CreateContext(Action configure) + { + var options = new DbContextOptionsBuilder() + .UseClickHouse(_connectionString) + .EnableServiceProviderCaching(false) + .Options; + return new TestContext(options, configure); + } + + private static async Task QueryScalar(DbContext context, string sql) + { + var conn = context.Database.GetDbConnection(); + await conn.OpenAsync(); + try + { + using var cmd = conn.CreateCommand(); + cmd.CommandText = sql; + var result = await cmd.ExecuteScalarAsync(); + return result?.ToString(); + } + finally + { + await conn.CloseAsync(); + } + } + + private class TestContext : DbContext + { + private readonly Action _configure; + + public TestContext(DbContextOptions options, Action configure) + : base(options) + { + _configure = configure; + } + + protected override void OnModelCreating(ModelBuilder modelBuilder) => _configure(modelBuilder); + } + + public class SimpleEntity + { + public long Id { get; set; } + public string Name { get; set; } = string.Empty; + } + + public class VersionedEntity + { + public long Id { get; set; } + public string Name { get; set; } = string.Empty; + public ulong Version { get; set; } + } + + public class TimestampedEntity + { + public long Id { get; set; } + public DateTime Timestamp { get; set; } + } +} diff --git a/test/EFCore.ClickHouse.Tests/MigrationSqlGeneratorTests.cs b/test/EFCore.ClickHouse.Tests/MigrationSqlGeneratorTests.cs new file mode 100644 index 0000000..0d5d2b6 --- /dev/null +++ b/test/EFCore.ClickHouse.Tests/MigrationSqlGeneratorTests.cs @@ -0,0 +1,559 @@ +using ClickHouse.EntityFrameworkCore.Metadata.Internal; +using ClickHouse.EntityFrameworkCore.Migrations.Operations; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Migrations.Operations; +using Xunit; + +namespace EFCore.ClickHouse.Tests; + +public class MigrationSqlGeneratorTests +{ + [Fact] + public void CreateTable_MergeTree_with_OrderBy() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.MergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id", "Name" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + op.Columns.Add(new AddColumnOperation { Name = "Name", ColumnType = "String", ClrType = typeof(string) }); + }); + + Assert.Contains("ENGINE = MergeTree()", sql); + Assert.Contains("ORDER BY (`Id`, `Name`)", sql); + } + + [Fact] + public void CreateTable_ReplacingMergeTree_with_version() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.ReplacingMergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.ReplacingMergeTreeVersion, "Version"); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + op.Columns.Add(new AddColumnOperation { Name = "Version", ColumnType = "UInt64", ClrType = typeof(ulong) }); + }); + + Assert.Contains("ENGINE = ReplacingMergeTree(`Version`)", sql); + } + + [Fact] + public void CreateTable_CollapsingMergeTree_with_sign() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.CollapsingMergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.CollapsingMergeTreeSign, "Sign"); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + op.Columns.Add(new AddColumnOperation { Name = "Sign", ColumnType = "Int8", ClrType = typeof(sbyte) }); + }); + + Assert.Contains("ENGINE = CollapsingMergeTree(`Sign`)", sql); + } + + [Fact] + public void CreateTable_StripeLog_no_OrderBy() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.StripeLog); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + }); + + Assert.Contains("ENGINE = StripeLog", sql); + Assert.DoesNotContain("StripeLog()", sql); + Assert.DoesNotContain("ORDER BY", sql); + } + + [Fact] + public void CreateTable_Memory_no_parentheses() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.Memory); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + }); + + Assert.Contains("ENGINE = Memory", sql); + Assert.DoesNotContain("Memory()", sql); + Assert.DoesNotContain("ORDER BY", sql); + } + + [Fact] + public void CreateTable_nullable_column_wraps_in_Nullable() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.MergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + op.Columns.Add(new AddColumnOperation + { + Name = "Value", ColumnType = "String", ClrType = typeof(string), IsNullable = true + }); + }); + + Assert.Contains("`Value` Nullable(String)", sql); + Assert.DoesNotContain("NOT NULL", sql); + } + + [Fact] + public void CreateTable_column_with_codec() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.MergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + var col = new AddColumnOperation { Name = "Temp", ColumnType = "Int16", ClrType = typeof(short) }; + col.AddAnnotation(ClickHouseAnnotationNames.ColumnCodec, "Delta, ZSTD"); + op.Columns.Add(col); + }); + + Assert.Contains("`Temp` Int16 CODEC(Delta, ZSTD)", sql); + } + + [Fact] + public void CreateTable_column_with_ttl() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.MergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + var col = new AddColumnOperation { Name = "Created", ColumnType = "DateTime", ClrType = typeof(DateTime) }; + col.AddAnnotation(ClickHouseAnnotationNames.ColumnTtl, "Created + INTERVAL 1 MONTH"); + op.Columns.Add(col); + }); + + Assert.Contains("TTL Created + INTERVAL 1 MONTH", sql); + } + + [Fact] + public void CreateTable_column_with_comment() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.MergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + var col = new AddColumnOperation { Name = "Name", ColumnType = "String", ClrType = typeof(string) }; + col.AddAnnotation(ClickHouseAnnotationNames.ColumnComment, "User name"); + op.Columns.Add(col); + }); + + Assert.Contains("COMMENT 'User name'", sql); + } + + [Fact] + public void CreateTable_with_partitionBy_expression() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.MergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id" }); + op.AddAnnotation(ClickHouseAnnotationNames.PartitionBy, new[] { "toYYYYMM(CreatedAt)" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + }); + + Assert.Contains("PARTITION BY toYYYYMM(CreatedAt)", sql); + } + + [Fact] + public void CreateTable_with_settings() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.MergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id" }); + op.AddAnnotation(ClickHouseAnnotationNames.SettingPrefix + "index_granularity", "4096"); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + }); + + Assert.Contains("SETTINGS index_granularity = 4096", sql); + } + + [Fact] + public void CreateTable_with_table_TTL() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.MergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id" }); + op.AddAnnotation(ClickHouseAnnotationNames.Ttl, "Created + INTERVAL 30 DAY"); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + }); + + Assert.Contains("TTL Created + INTERVAL 30 DAY", sql); + } + + [Fact] + public void CreateTable_MergeTree_no_explicit_OrderBy_falls_back_to_tuple() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.MergeTree); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + }); + + Assert.Contains("ORDER BY tuple()", sql); + } + + [Fact] + public void AddForeignKey_throws_NotSupportedException() + { + Assert.Throws(() => + { + Generate(new AddForeignKeyOperation + { + Table = "t", + Name = "FK_Test", + Columns = ["Id"], + PrincipalTable = "other", + PrincipalColumns = ["Id"] + }); + }); + } + + [Fact] + public void CreateSequence_throws_NotSupportedException() + { + Assert.Throws(() => + { + Generate(new CreateSequenceOperation { Name = "seq" }); + }); + } + + [Fact] + public void EnsureSchema_throws_NotSupportedException() + { + Assert.Throws(() => + { + Generate(new EnsureSchemaOperation { Name = "dbo" }); + }); + } + + [Fact] + public void RenameTable_generates_RENAME_TABLE() + { + var sql = Generate(new RenameTableOperation { Name = "old_table", NewName = "new_table" }); + Assert.Contains("RENAME TABLE `old_table` TO `new_table`", sql); + } + + [Fact] + public void RenameColumn_generates_ALTER_TABLE_RENAME_COLUMN() + { + var sql = Generate(new RenameColumnOperation { Table = "t", Name = "old_col", NewName = "new_col" }); + Assert.Contains("ALTER TABLE `t` RENAME COLUMN `old_col` TO `new_col`", sql); + } + + [Fact] + public void DropIndex_skipping_generates_ALTER_TABLE_DROP_INDEX() + { + var op = new DropIndexOperation { Table = "t", Name = "idx_name" }; + op.AddAnnotation(ClickHouseAnnotationNames.SkippingIndexType, "minmax"); + var sql = Generate(op); + Assert.Contains("ALTER TABLE `t` DROP INDEX `idx_name`", sql); + } + + // Finding 3: standard index create/drop symmetry + + [Fact] + public void CreateIndex_standard_is_noop() + { + var op = new CreateIndexOperation + { + Name = "IX_Test", Table = "t", Columns = ["Col1"] + }; + var sql = Generate(op); + Assert.DoesNotContain("INDEX", sql); + } + + [Fact] + public void DropIndex_standard_is_noop() + { + var op = new DropIndexOperation { Table = "t", Name = "IX_Test" }; + // No skipping index annotation — should be no-op, symmetric with create + var sql = Generate(op); + Assert.DoesNotContain("INDEX", sql); + } + + // Finding 1: AlterTableOperation rejects ClickHouse metadata changes + + [Fact] + public void AlterTable_engine_change_throws() + { + var op = new AlterTableOperation { Name = "t" }; + op.AddAnnotation(ClickHouseAnnotationNames.Engine, "ReplacingMergeTree"); + op.OldTable.AddAnnotation(ClickHouseAnnotationNames.Engine, "MergeTree"); + + Assert.Throws(() => Generate(op)); + } + + [Fact] + public void AlterTable_orderBy_change_throws() + { + var op = new AlterTableOperation { Name = "t" }; + op.AddAnnotation(ClickHouseAnnotationNames.Engine, "MergeTree"); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id", "Name" }); + op.OldTable.AddAnnotation(ClickHouseAnnotationNames.Engine, "MergeTree"); + op.OldTable.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id" }); + + Assert.Throws(() => Generate(op)); + } + + [Fact] + public void AlterTable_partitionBy_change_throws() + { + var op = new AlterTableOperation { Name = "t" }; + op.AddAnnotation(ClickHouseAnnotationNames.Engine, "MergeTree"); + op.AddAnnotation(ClickHouseAnnotationNames.PartitionBy, new[] { "toYYYYMM(ts)" }); + op.OldTable.AddAnnotation(ClickHouseAnnotationNames.Engine, "MergeTree"); + + Assert.Throws(() => Generate(op)); + } + + [Fact] + public void AlterTable_ttl_change_throws() + { + var op = new AlterTableOperation { Name = "t" }; + op.AddAnnotation(ClickHouseAnnotationNames.Ttl, "ts + INTERVAL 30 DAY"); + op.OldTable.AddAnnotation(ClickHouseAnnotationNames.Ttl, "ts + INTERVAL 7 DAY"); + + Assert.Throws(() => Generate(op)); + } + + [Fact] + public void AlterTable_primaryKey_change_throws() + { + var op = new AlterTableOperation { Name = "t" }; + op.AddAnnotation(ClickHouseAnnotationNames.Engine, "MergeTree"); + op.AddAnnotation(ClickHouseAnnotationNames.PrimaryKey, new[] { "Id", "Name" }); + op.OldTable.AddAnnotation(ClickHouseAnnotationNames.Engine, "MergeTree"); + op.OldTable.AddAnnotation(ClickHouseAnnotationNames.PrimaryKey, new[] { "Id" }); + + Assert.Throws(() => Generate(op)); + } + + [Fact] + public void AlterTable_sampleBy_change_throws() + { + var op = new AlterTableOperation { Name = "t" }; + op.AddAnnotation(ClickHouseAnnotationNames.SampleBy, new[] { "Id" }); + op.OldTable.AddAnnotation(ClickHouseAnnotationNames.Engine, "MergeTree"); + + Assert.Throws(() => Generate(op)); + } + + [Fact] + public void AlterTable_settings_change_throws() + { + var op = new AlterTableOperation { Name = "t" }; + op.AddAnnotation(ClickHouseAnnotationNames.SettingPrefix + "index_granularity", "4096"); + op.OldTable.AddAnnotation(ClickHouseAnnotationNames.SettingPrefix + "index_granularity", "8192"); + + Assert.Throws(() => Generate(op)); + } + + [Fact] + public void AlterTable_engine_specific_arg_change_throws() + { + var op = new AlterTableOperation { Name = "t" }; + op.AddAnnotation(ClickHouseAnnotationNames.Engine, "ReplacingMergeTree"); + op.AddAnnotation(ClickHouseAnnotationNames.ReplacingMergeTreeVersion, "Ver2"); + op.OldTable.AddAnnotation(ClickHouseAnnotationNames.Engine, "ReplacingMergeTree"); + op.OldTable.AddAnnotation(ClickHouseAnnotationNames.ReplacingMergeTreeVersion, "Ver1"); + + Assert.Throws(() => Generate(op)); + } + + [Fact] + public void AlterTable_no_clickhouse_changes_delegates_to_base() + { + // Non-ClickHouse metadata change (e.g., comment) should not throw + var op = new AlterTableOperation { Name = "t", Comment = "new comment" }; + op.OldTable.Comment = "old comment"; + var sql = Generate(op); + // Should not throw — base handles standard annotation changes + Assert.NotNull(sql); + } + + // Finding 2: idempotent scripts throw + + [Fact] + public void GetBeginIfNotExistsScript_throws_NotSupportedException() + { + var repo = CreateHistoryRepository(); + Assert.Throws(() => repo.GetBeginIfNotExistsScript("20260101000000_Init")); + } + + [Fact] + public void GetBeginIfExistsScript_throws_NotSupportedException() + { + var repo = CreateHistoryRepository(); + Assert.Throws(() => repo.GetBeginIfExistsScript("20260101000000_Init")); + } + + [Fact] + public void GetEndIfScript_throws_NotSupportedException() + { + var repo = CreateHistoryRepository(); + Assert.Throws(() => repo.GetEndIfScript()); + } + + [Fact] + public void GetCreateIfNotExistsScript_contains_IF_NOT_EXISTS() + { + var repo = CreateHistoryRepository(); + var script = repo.GetCreateIfNotExistsScript(); + Assert.Contains("IF NOT EXISTS", script); + Assert.Contains("CREATE TABLE", script); + } + + // Corner cases from review section D + + [Fact] + public void Column_comment_with_single_quote_is_escaped() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.MergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + var col = new AddColumnOperation { Name = "Name", ColumnType = "String", ClrType = typeof(string) }; + col.AddAnnotation(ClickHouseAnnotationNames.ColumnComment, "it's a name"); + op.Columns.Add(col); + }); + + Assert.Contains(@"COMMENT 'it\'s a name'", sql); + } + + [Fact] + public void Column_with_codec_ttl_and_comment_together() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.MergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + var col = new AddColumnOperation { Name = "Temp", ColumnType = "Int16", ClrType = typeof(short) }; + col.AddAnnotation(ClickHouseAnnotationNames.ColumnCodec, "Delta, ZSTD"); + col.AddAnnotation(ClickHouseAnnotationNames.ColumnTtl, "ts + INTERVAL 1 DAY"); + col.AddAnnotation(ClickHouseAnnotationNames.ColumnComment, "temperature"); + op.Columns.Add(col); + }); + + // Verify order: type CODEC TTL COMMENT + var tempLine = sql.Split('\n').First(l => l.Contains("`Temp`")); + var codecIdx = tempLine.IndexOf("CODEC(", StringComparison.Ordinal); + var ttlIdx = tempLine.IndexOf("TTL ", StringComparison.Ordinal); + var commentIdx = tempLine.IndexOf("COMMENT ", StringComparison.Ordinal); + Assert.True(codecIdx < ttlIdx, "CODEC should come before TTL"); + Assert.True(ttlIdx < commentIdx, "TTL should come before COMMENT"); + } + + [Fact] + public void Nullable_array_column_is_not_wrapped_in_Nullable() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.MergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + op.Columns.Add(new AddColumnOperation + { + Name = "Tags", ColumnType = "Array(String)", ClrType = typeof(string[]), IsNullable = true + }); + }); + + Assert.Contains("`Tags` Array(String)", sql); + Assert.DoesNotContain("Nullable(Array", sql); + } + + [Fact] + public void OrderBy_mixed_expressions_and_columns() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.MergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id", "toYYYYMM(ts)" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + }); + + Assert.Contains("ORDER BY (`Id`, toYYYYMM(ts))", sql); + } + + [Fact] + public void VersionedCollapsingMergeTree_both_args() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.VersionedCollapsingMergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.VersionedCollapsingMergeTreeSign, "Sign"); + op.AddAnnotation(ClickHouseAnnotationNames.VersionedCollapsingMergeTreeVersion, "Ver"); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + }); + + Assert.Contains("ENGINE = VersionedCollapsingMergeTree(`Sign`, `Ver`)", sql); + } + + [Fact] + public void SummingMergeTree_multiple_columns() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.SummingMergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.SummingMergeTreeColumns, new[] { "Amount", "Count" }); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + }); + + Assert.Contains("ENGINE = SummingMergeTree(`Amount`, `Count`)", sql); + } + + [Fact] + public void CreateDatabase_generates_CREATE_DATABASE() + { + var sql = Generate(new ClickHouseCreateDatabaseOperation { Name = "my_db" }); + Assert.Contains("CREATE DATABASE `my_db`", sql); + } + + [Fact] + public void DropDatabase_generates_DROP_DATABASE() + { + var sql = Generate(new ClickHouseDropDatabaseOperation { Name = "my_db" }); + Assert.Contains("DROP DATABASE `my_db`", sql); + } + + private string GenerateCreateTable(Action configure) + { + var operation = new CreateTableOperation { Name = "test_table" }; + configure(operation); + return Generate(operation); + } + + private string Generate(params MigrationOperation[] operations) + { + var optionsBuilder = new DbContextOptionsBuilder() + .UseClickHouse("Host=localhost;Database=test"); + + using var context = new DbContext(optionsBuilder.Options); + var generator = context.GetService(); + var commands = generator.Generate(operations); + return string.Join("\n", commands.Select(c => c.CommandText)); + } + + private static IHistoryRepository CreateHistoryRepository() + { + var optionsBuilder = new DbContextOptionsBuilder() + .UseClickHouse("Host=localhost;Database=test"); + + using var context = new DbContext(optionsBuilder.Options); + return context.GetService(); + } +} From 2ec0c818f11060ae9edaadbfb15baa0edde29123 Mon Sep 17 00:00:00 2001 From: Alex Soffronow-Pagonidis Date: Fri, 10 Apr 2026 21:37:05 +0200 Subject: [PATCH 2/2] integration test adjustments --- .github/workflows/ef-cli-smoke.yml | 73 +-------- .../ClickHouseMigrationsSqlGenerator.cs | 20 +-- .../SmokeDbContext.cs | 10 +- .../DotnetEfCliTests.cs | 144 ++++++++++++++++++ .../MigrationSqlGeneratorTests.cs | 6 +- 5 files changed, 166 insertions(+), 87 deletions(-) create mode 100644 test/EFCore.ClickHouse.Tests/DotnetEfCliTests.cs diff --git a/.github/workflows/ef-cli-smoke.yml b/.github/workflows/ef-cli-smoke.yml index 3250992..5645013 100644 --- a/.github/workflows/ef-cli-smoke.yml +++ b/.github/workflows/ef-cli-smoke.yml @@ -10,7 +10,7 @@ on: jobs: ef-cli-smoke: - name: dotnet-ef smoke test + name: dotnet-ef integration tests runs-on: ubuntu-latest steps: - name: Checkout @@ -24,72 +24,5 @@ jobs: - name: Install dotnet-ef run: dotnet tool install --global dotnet-ef - - name: Restore & Build - run: dotnet build test/EFCore.ClickHouse.DesignSmoke/EFCore.ClickHouse.DesignSmoke.csproj - - - name: Add migration - run: | - dotnet-ef migrations add InitialCreate \ - --project test/EFCore.ClickHouse.DesignSmoke \ - --startup-project test/EFCore.ClickHouse.DesignSmoke - - - name: Verify migration contains ClickHouse annotations - run: | - MIGRATION_FILE=$(find test/EFCore.ClickHouse.DesignSmoke/Migrations -name '*_InitialCreate.cs' ! -name '*.Designer.cs') - echo "Checking $MIGRATION_FILE" - - grep -F 'ClickHouse:Engine' "$MIGRATION_FILE" - grep -F 'ClickHouse:OrderBy' "$MIGRATION_FILE" - grep -F 'ClickHouse:PartitionBy' "$MIGRATION_FILE" - grep -F 'ClickHouse:PrimaryKey' "$MIGRATION_FILE" - grep -F 'ClickHouse:SampleBy' "$MIGRATION_FILE" - grep -F 'ClickHouse:Setting:index_granularity' "$MIGRATION_FILE" - grep -F 'ClickHouse:Ttl' "$MIGRATION_FILE" - grep -F 'ClickHouse:ColumnCodec' "$MIGRATION_FILE" - grep -F 'ClickHouse:ColumnTtl' "$MIGRATION_FILE" - grep -F 'ClickHouse:ColumnComment' "$MIGRATION_FILE" - grep -F 'ClickHouse:SkippingIndex:Type' "$MIGRATION_FILE" - grep -F 'ClickHouse:SkippingIndex:Granularity' "$MIGRATION_FILE" - echo "All ClickHouse annotations present in migration." - - - name: Generate SQL script - run: | - dotnet-ef migrations script \ - --project test/EFCore.ClickHouse.DesignSmoke \ - --startup-project test/EFCore.ClickHouse.DesignSmoke \ - > migration.sql - cat migration.sql - - - name: Verify SQL script content - run: | - # Engine and clauses - grep -F 'ENGINE = ReplacingMergeTree' migration.sql - grep -F 'ORDER BY' migration.sql - grep -F 'PARTITION BY' migration.sql - grep -F 'PRIMARY KEY' migration.sql - grep -F 'SAMPLE BY' migration.sql - grep -F 'TTL Timestamp + INTERVAL 1 YEAR' migration.sql - grep -F 'SETTINGS index_granularity = 4096' migration.sql - grep -F 'ADD INDEX' migration.sql - grep -F 'CODEC(Delta, ZSTD)' migration.sql - - # Simple engine without parentheses - grep -F 'ENGINE = Memory' migration.sql - ! grep -F 'ENGINE = Memory()' migration.sql - - # No transaction wrapping (ClickHouse does not support transactions) - ! grep -F 'START TRANSACTION' migration.sql - ! grep -F 'COMMIT' migration.sql - - echo "SQL script content verified." - - - name: Verify idempotent script is rejected - run: | - if dotnet-ef migrations script --idempotent \ - --project test/EFCore.ClickHouse.DesignSmoke \ - --startup-project test/EFCore.ClickHouse.DesignSmoke 2>&1; then - echo "ERROR: --idempotent should have failed" - exit 1 - else - echo "Idempotent script correctly rejected." - fi + - name: Run dotnet-ef CLI tests + run: dotnet test test/EFCore.ClickHouse.Tests/EFCore.ClickHouse.Tests.csproj --filter "FullyQualifiedName~DotnetEfCliTests" diff --git a/src/EFCore.ClickHouse/Migrations/ClickHouseMigrationsSqlGenerator.cs b/src/EFCore.ClickHouse/Migrations/ClickHouseMigrationsSqlGenerator.cs index 89432dd..c7c05cb 100644 --- a/src/EFCore.ClickHouse/Migrations/ClickHouseMigrationsSqlGenerator.cs +++ b/src/EFCore.ClickHouse/Migrations/ClickHouseMigrationsSqlGenerator.cs @@ -112,23 +112,25 @@ protected override void ColumnDefinition( if (!string.IsNullOrWhiteSpace(defaultValue)) builder.Append(" DEFAULT ").Append(defaultValue); + // ClickHouse column definition order: DEFAULT → COMMENT → CODEC → TTL + + // COMMENT + var comment = operation.FindAnnotation(ClickHouseAnnotationNames.ColumnComment); + if (comment?.Value is string commentStr && !string.IsNullOrWhiteSpace(commentStr)) + { + var escaped = commentStr.Replace("'", "\\'"); + builder.Append($" COMMENT '{escaped}'"); + } + // CODEC var codec = operation.FindAnnotation(ClickHouseAnnotationNames.ColumnCodec); if (codec?.Value is string codecStr && !string.IsNullOrWhiteSpace(codecStr)) builder.Append($" CODEC({codecStr})"); - // Column TTL + // TTL var columnTtl = operation.FindAnnotation(ClickHouseAnnotationNames.ColumnTtl); if (columnTtl?.Value is string ttlStr && !string.IsNullOrWhiteSpace(ttlStr)) builder.Append($" TTL {ttlStr}"); - - // Column COMMENT - var comment = operation.FindAnnotation(ClickHouseAnnotationNames.ColumnComment); - if (comment?.Value is string commentStr && !string.IsNullOrWhiteSpace(commentStr)) - { - var escaped = commentStr.Replace("'", "\\'"); - builder.Append($" COMMENT '{escaped}'"); - } } protected override void ComputedColumnDefinition( diff --git a/test/EFCore.ClickHouse.DesignSmoke/SmokeDbContext.cs b/test/EFCore.ClickHouse.DesignSmoke/SmokeDbContext.cs index fa9ed37..e1a4b98 100644 --- a/test/EFCore.ClickHouse.DesignSmoke/SmokeDbContext.cs +++ b/test/EFCore.ClickHouse.DesignSmoke/SmokeDbContext.cs @@ -18,17 +18,15 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) b.HasKey(e => e.Id); b.Property(e => e.Temperature).HasCodec("Delta, ZSTD"); b.Property(e => e.Timestamp) - .HasColumnTtl("Timestamp + INTERVAL 90 DAY") .HasColumnComment("Reading timestamp"); b.HasIndex(e => e.Timestamp) .HasSkippingIndexType("minmax") .HasGranularity(4); b.ToTable("sensor_readings", t => t .HasReplacingMergeTreeEngine("Version") - .WithOrderBy("SensorId", "Timestamp") + .WithOrderBy("Id", "Timestamp") .WithPartitionBy("toYYYYMM(Timestamp)") - .WithPrimaryKey("SensorId") - .WithSampleBy("SensorId") + .WithPrimaryKey("Id") .WithTtl("Timestamp + INTERVAL 1 YEAR") .WithSetting("index_granularity", "4096")); }); @@ -60,8 +58,10 @@ public class SmokeDbContextFactory : IDesignTimeDbContextFactory { public SmokeDbContext CreateDbContext(string[] args) { + var connectionString = Environment.GetEnvironmentVariable("CLICKHOUSE_CONNECTION_STRING") + ?? "Host=localhost;Database=smoke_test"; var optionsBuilder = new DbContextOptionsBuilder(); - optionsBuilder.UseClickHouse("Host=localhost;Database=smoke_test"); + optionsBuilder.UseClickHouse(connectionString); return new SmokeDbContext(optionsBuilder.Options); } } diff --git a/test/EFCore.ClickHouse.Tests/DotnetEfCliTests.cs b/test/EFCore.ClickHouse.Tests/DotnetEfCliTests.cs new file mode 100644 index 0000000..bd7ecec --- /dev/null +++ b/test/EFCore.ClickHouse.Tests/DotnetEfCliTests.cs @@ -0,0 +1,144 @@ +using System.Diagnostics; +using Xunit; + +namespace EFCore.ClickHouse.Tests; + +/// +/// End-to-end tests that shell out to the real dotnet-ef CLI tool +/// and verify the resulting database state against a real ClickHouse instance. +/// +public class DotnetEfCliTests : IAsyncLifetime +{ + private string _connectionString = default!; + private string _smokeProjectDir = default!; + private string? _migrationsDir; + + public async Task InitializeAsync() + { + _connectionString = await SharedContainer.GetConnectionStringAsync(); + + var testDir = Path.GetDirectoryName(typeof(DotnetEfCliTests).Assembly.Location)!; + var repoRoot = Path.GetFullPath(Path.Combine(testDir, "..", "..", "..", "..", "..")); + _smokeProjectDir = Path.Combine(repoRoot, "test", "EFCore.ClickHouse.DesignSmoke"); + + Assert.True(Directory.Exists(_smokeProjectDir), + $"DesignSmoke project not found at {_smokeProjectDir}"); + + _migrationsDir = Path.Combine(_smokeProjectDir, "Migrations"); + if (Directory.Exists(_migrationsDir)) + Directory.Delete(_migrationsDir, recursive: true); + } + + public Task DisposeAsync() + { + if (_migrationsDir is not null && Directory.Exists(_migrationsDir)) + Directory.Delete(_migrationsDir, recursive: true); + return Task.CompletedTask; + } + + [Fact] + public async Task Database_update_creates_correct_schema() + { + // Add migration and apply to real ClickHouse + await RunDotnetEfSuccessfully("migrations", "add", "InitialCreate"); + await RunDotnetEfSuccessfully("database", "update"); + + // Verify everything via ClickHouse system tables + using var connection = new global::ClickHouse.Driver.ADO.ClickHouseConnection(_connectionString); + await connection.OpenAsync(); + + // History table tracks the migration + var historyCount = await QueryScalar(connection, + "SELECT count() FROM `__EFMigrationsHistory`"); + Assert.Equal(1UL, historyCount); + + // sensor_readings created with ReplacingMergeTree + var sensorEngine = await QueryScalar(connection, + "SELECT engine FROM system.tables WHERE name = 'sensor_readings'"); + Assert.Equal("ReplacingMergeTree", sensorEngine); + + // audit_logs created with Memory + var auditEngine = await QueryScalar(connection, + "SELECT engine FROM system.tables WHERE name = 'audit_logs'"); + Assert.Equal("Memory", auditEngine); + + // ORDER BY / sorting key + var sortingKey = await QueryScalar(connection, + "SELECT sorting_key FROM system.tables WHERE name = 'sensor_readings'"); + Assert.Contains("Id", sortingKey); + Assert.Contains("Timestamp", sortingKey); + + // PARTITION BY + var partitionKey = await QueryScalar(connection, + "SELECT partition_key FROM system.tables WHERE name = 'sensor_readings'"); + Assert.Contains("toYYYYMM(Timestamp)", partitionKey); + + // PRIMARY KEY + var primaryKey = await QueryScalar(connection, + "SELECT primary_key FROM system.tables WHERE name = 'sensor_readings'"); + Assert.Contains("Id", primaryKey); + + // Data skipping index exists + var indexCount = await QueryScalar(connection, + "SELECT count() FROM system.data_skipping_indices WHERE table = 'sensor_readings'"); + Assert.True(indexCount > 0, "Expected at least one data skipping index"); + } + + [Fact] + public async Task Idempotent_script_is_rejected() + { + await RunDotnetEfSuccessfully("migrations", "add", "InitialCreate"); + + var result = await RunDotnetEf("migrations", "script", "--idempotent"); + Assert.True(result.ExitCode != 0, "Expected --idempotent to fail"); + var output = result.StdOut + result.StdErr; + Assert.Contains("does not support conditional SQL blocks", output); + } + + private async Task RunDotnetEfSuccessfully(params string[] args) + { + var result = await RunDotnetEf(args); + Assert.True(result.ExitCode == 0, + $"dotnet-ef {string.Join(' ', args)} failed (exit {result.ExitCode}):\n{result.StdOut}\n{result.StdErr}"); + } + + private async Task RunDotnetEf(params string[] args) + { + var allArgs = new List(args) + { + "--project", _smokeProjectDir, + "--startup-project", _smokeProjectDir + }; + + var psi = new ProcessStartInfo + { + FileName = "dotnet-ef", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + WorkingDirectory = _smokeProjectDir + }; + psi.Environment["CLICKHOUSE_CONNECTION_STRING"] = _connectionString; + + foreach (var arg in allArgs) + psi.ArgumentList.Add(arg); + + using var process = Process.Start(psi)!; + var stdout = await process.StandardOutput.ReadToEndAsync(); + var stderr = await process.StandardError.ReadToEndAsync(); + await process.WaitForExitAsync(); + + return new DotnetEfResult(process.ExitCode, stdout, stderr); + } + + private static async Task QueryScalar( + global::ClickHouse.Driver.ADO.ClickHouseConnection connection, string sql) + { + using var cmd = connection.CreateCommand(); + cmd.CommandText = sql; + var result = await cmd.ExecuteScalarAsync(); + return (T)Convert.ChangeType(result!, typeof(T)); + } + + private record DotnetEfResult(int ExitCode, string StdOut, string StdErr); +} diff --git a/test/EFCore.ClickHouse.Tests/MigrationSqlGeneratorTests.cs b/test/EFCore.ClickHouse.Tests/MigrationSqlGeneratorTests.cs index 0d5d2b6..116ce8c 100644 --- a/test/EFCore.ClickHouse.Tests/MigrationSqlGeneratorTests.cs +++ b/test/EFCore.ClickHouse.Tests/MigrationSqlGeneratorTests.cs @@ -447,13 +447,13 @@ public void Column_with_codec_ttl_and_comment_together() op.Columns.Add(col); }); - // Verify order: type CODEC TTL COMMENT + // Verify order per ClickHouse docs: COMMENT → CODEC → TTL var tempLine = sql.Split('\n').First(l => l.Contains("`Temp`")); + var commentIdx = tempLine.IndexOf("COMMENT ", StringComparison.Ordinal); var codecIdx = tempLine.IndexOf("CODEC(", StringComparison.Ordinal); var ttlIdx = tempLine.IndexOf("TTL ", StringComparison.Ordinal); - var commentIdx = tempLine.IndexOf("COMMENT ", StringComparison.Ordinal); + Assert.True(commentIdx < codecIdx, "COMMENT should come before CODEC"); Assert.True(codecIdx < ttlIdx, "CODEC should come before TTL"); - Assert.True(ttlIdx < commentIdx, "TTL should come before COMMENT"); } [Fact]