From 3a50dd43a595575083133ace61285cec235d288b Mon Sep 17 00:00:00 2001 From: mblos Date: Mon, 13 Apr 2026 15:03:52 +0200 Subject: [PATCH 1/2] WM-343 remove nova client from CR usage API --- cmd/manager/main.go | 17 +-- helm/bundles/cortex-nova/values.yaml | 3 + .../reservations/commitments/api.go | 35 ++++-- .../commitments/api_report_usage.go | 2 +- .../commitments/api_report_usage_test.go | 62 +++++----- .../reservations/commitments/controller.go | 6 + .../reservations/commitments/usage.go | 116 +++++++++--------- .../reservations/commitments/usage_db.go | 65 ++++++++++ .../reservations/commitments/usage_test.go | 102 +++++++-------- 9 files changed, 240 insertions(+), 168 deletions(-) create mode 100644 internal/scheduling/reservations/commitments/usage_db.go diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 6ca021167..f5f3e1f0f 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -346,6 +346,11 @@ func main() { detectorPipelineMonitor := schedulinglib.NewDetectorPipelineMonitor() metrics.Registry.MustRegister(&detectorPipelineMonitor) + // Initialize commitments API for LIQUID interface (Postgres-backed usage reporting). + commitmentsConfig := conf.GetConfigOrDie[commitments.Config]() + commitmentsAPI := commitments.NewAPIWithConfig(multiclusterClient, commitmentsConfig, nil) + commitmentsAPI.Init(mux, metrics.Registry, ctrl.Log.WithName("commitments-api")) + if slices.Contains(mainConfig.EnabledControllers, "nova-pipeline-controllers") { // Filter-weigher pipeline controller setup. filterWeigherController := &nova.FilterWeigherPipelineController{ @@ -373,11 +378,6 @@ func main() { os.Exit(1) } - // Initialize commitments API for LIQUID interface (with Nova client for usage reporting) - commitmentsConfig := conf.GetConfigOrDie[commitments.Config]() - commitmentsAPI := commitments.NewAPIWithConfig(multiclusterClient, commitmentsConfig, novaClient) - commitmentsAPI.Init(mux, metrics.Registry, ctrl.Log.WithName("commitments-api")) - deschedulingsController := &nova.DetectorPipelineController{ Monitor: detectorPipelineMonitor, Breaker: &nova.DetectorCycleBreaker{NovaClient: novaClient}, @@ -522,9 +522,10 @@ func main() { commitmentsConfig.ApplyDefaults() if err := (&commitments.CommitmentReservationController{ - Client: multiclusterClient, - Scheme: mgr.GetScheme(), - Conf: commitmentsConfig, + Client: multiclusterClient, + Scheme: mgr.GetScheme(), + Conf: commitmentsConfig, + UsageAPI: commitmentsAPI, }).SetupWithManager(mgr, multiclusterClient); err != nil { setupLog.Error(err, "unable to create controller", "controller", "CommitmentReservation") os.Exit(1) diff --git a/helm/bundles/cortex-nova/values.yaml b/helm/bundles/cortex-nova/values.yaml index d32316e47..d30d89a4f 100644 --- a/helm/bundles/cortex-nova/values.yaml +++ b/helm/bundles/cortex-nova/values.yaml @@ -99,6 +99,9 @@ cortex: &cortex keystoneSecretRef: name: cortex-nova-openstack-keystone namespace: default + databaseSecretRef: + name: cortex-nova-postgres + namespace: default # ssoSecretRef: # name: cortex-nova-openstack-sso # namespace: default diff --git a/internal/scheduling/reservations/commitments/api.go b/internal/scheduling/reservations/commitments/api.go index 06fb97be1..ff32a413f 100644 --- a/internal/scheduling/reservations/commitments/api.go +++ b/internal/scheduling/reservations/commitments/api.go @@ -9,23 +9,37 @@ import ( "strings" "sync" - "github.com/cobaltcore-dev/cortex/internal/scheduling/nova" "github.com/go-logr/logr" "github.com/prometheus/client_golang/prometheus" "sigs.k8s.io/controller-runtime/pkg/client" ) -// UsageNovaClient is a minimal interface for the Nova client needed by the usage API. -// This allows for easy mocking in tests without implementing the full NovaClient interface. -type UsageNovaClient interface { - ListProjectServers(ctx context.Context, projectID string) ([]nova.ServerDetail, error) +// UsageDBClient is the minimal interface for querying VM usage data from Postgres. +type UsageDBClient interface { + // ListProjectVMs returns all VMs for a project with their flavor data. + ListProjectVMs(ctx context.Context, projectID string) ([]VMRow, error) +} + +// VMRow is the result of a joined server+flavor query from Postgres. +type VMRow struct { + ID string + Name string + Status string + Created string + AZ string + Hypervisor string + FlavorName string + FlavorRAM uint64 + FlavorVCPUs uint64 + FlavorDisk uint64 + FlavorExtras string // JSON string of flavor extra_specs } // HTTPAPI implements Limes LIQUID commitment validation endpoints. type HTTPAPI struct { client client.Client config Config - novaClient UsageNovaClient + usageDB UsageDBClient monitor ChangeCommitmentsAPIMonitor usageMonitor ReportUsageAPIMonitor capacityMonitor ReportCapacityAPIMonitor @@ -38,11 +52,11 @@ func NewAPI(client client.Client) *HTTPAPI { return NewAPIWithConfig(client, DefaultConfig(), nil) } -func NewAPIWithConfig(client client.Client, config Config, novaClient UsageNovaClient) *HTTPAPI { +func NewAPIWithConfig(client client.Client, config Config, usageDB UsageDBClient) *HTTPAPI { return &HTTPAPI{ client: client, config: config, - novaClient: novaClient, + usageDB: usageDB, monitor: NewChangeCommitmentsAPIMonitor(), usageMonitor: NewReportUsageAPIMonitor(), capacityMonitor: NewReportCapacityAPIMonitor(), @@ -50,6 +64,11 @@ func NewAPIWithConfig(client client.Client, config Config, novaClient UsageNovaC } } +// SetUsageDB sets the UsageDBClient after construction (e.g. once the K8s cache is ready). +func (api *HTTPAPI) SetUsageDB(usageDB UsageDBClient) { + api.usageDB = usageDB +} + func (api *HTTPAPI) Init(mux *http.ServeMux, registry prometheus.Registerer, log logr.Logger) { registry.MustRegister(&api.monitor) registry.MustRegister(&api.usageMonitor) diff --git a/internal/scheduling/reservations/commitments/api_report_usage.go b/internal/scheduling/reservations/commitments/api_report_usage.go index f54917e1a..e0d39f81e 100644 --- a/internal/scheduling/reservations/commitments/api_report_usage.go +++ b/internal/scheduling/reservations/commitments/api_report_usage.go @@ -72,7 +72,7 @@ func (api *HTTPAPI) HandleReportUsage(w http.ResponseWriter, r *http.Request) { } // Use UsageCalculator to build usage report - calculator := NewUsageCalculator(api.client, api.novaClient) + calculator := NewUsageCalculator(api.client, api.usageDB) report, err := calculator.CalculateUsage(r.Context(), log, projectID, req.AllAZs) if err != nil { log.Error(err, "failed to calculate usage report", "projectID", projectID) diff --git a/internal/scheduling/reservations/commitments/api_report_usage_test.go b/internal/scheduling/reservations/commitments/api_report_usage_test.go index f54d2a679..77ecf472e 100644 --- a/internal/scheduling/reservations/commitments/api_report_usage_test.go +++ b/internal/scheduling/reservations/commitments/api_report_usage_test.go @@ -18,7 +18,6 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" - "github.com/cobaltcore-dev/cortex/internal/scheduling/nova" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "github.com/prometheus/client_golang/prometheus" "github.com/sapcc/go-api-declarations/liquid" @@ -432,42 +431,41 @@ type ExpectedVMUsage struct { } // ============================================================================ -// Mock Nova Client +// Mock UsageDBClient // ============================================================================ -type mockUsageNovaClient struct { - servers map[string][]nova.ServerDetail // projectID -> servers - err error +type mockUsageDBClient struct { + rows map[string][]VMRow // projectID -> rows + err error } -func newMockUsageNovaClient() *mockUsageNovaClient { - return &mockUsageNovaClient{ - servers: make(map[string][]nova.ServerDetail), +func newMockUsageDBClient() *mockUsageDBClient { + return &mockUsageDBClient{ + rows: make(map[string][]VMRow), } } -func (m *mockUsageNovaClient) ListProjectServers(_ context.Context, projectID string) ([]nova.ServerDetail, error) { +func (m *mockUsageDBClient) ListProjectVMs(_ context.Context, projectID string) ([]VMRow, error) { if m.err != nil { return nil, m.err } - return m.servers[projectID], nil + return m.rows[projectID], nil } -func (m *mockUsageNovaClient) addVM(vm *TestVMUsage) { - server := nova.ServerDetail{ - ID: vm.UUID, - Name: vm.UUID, - Status: "ACTIVE", - TenantID: vm.ProjectID, - Created: vm.CreatedAt.Format(time.RFC3339), - AvailabilityZone: vm.AZ, - Hypervisor: vm.Host, - FlavorName: vm.Flavor.Name, - FlavorRAM: uint64(vm.Flavor.MemoryMB), //nolint:gosec - FlavorVCPUs: uint64(vm.Flavor.VCPUs), //nolint:gosec - FlavorDisk: vm.Flavor.DiskGB, - } - m.servers[vm.ProjectID] = append(m.servers[vm.ProjectID], server) +func (m *mockUsageDBClient) addVM(vm *TestVMUsage) { + row := VMRow{ + ID: vm.UUID, + Name: vm.UUID, + Status: "ACTIVE", + Created: vm.CreatedAt.Format(time.RFC3339), + AZ: vm.AZ, + Hypervisor: vm.Host, + FlavorName: vm.Flavor.Name, + FlavorRAM: uint64(vm.Flavor.MemoryMB), //nolint:gosec + FlavorVCPUs: uint64(vm.Flavor.VCPUs), //nolint:gosec + FlavorDisk: vm.Flavor.DiskGB, + } + m.rows[vm.ProjectID] = append(m.rows[vm.ProjectID], row) } // ============================================================================ @@ -478,7 +476,7 @@ type UsageTestEnv struct { T *testing.T Scheme *runtime.Scheme K8sClient client.Client - NovaClient *mockUsageNovaClient + DBClient *mockUsageDBClient FlavorGroups FlavorGroupsKnowledge HTTPServer *httptest.Server API *HTTPAPI @@ -530,14 +528,14 @@ func newUsageTestEnv( }). Build() - // Create mock Nova client with VMs - novaClient := newMockUsageNovaClient() + // Create mock DB client with VMs + dbClient := newMockUsageDBClient() for _, vm := range vms { - novaClient.addVM(vm) + dbClient.addVM(vm) } - // Create API with mock Nova client - api := NewAPIWithConfig(k8sClient, DefaultConfig(), novaClient) + // Create API with mock DB client + api := NewAPIWithConfig(k8sClient, DefaultConfig(), dbClient) mux := http.NewServeMux() registry := prometheus.NewRegistry() api.Init(mux, registry, log.Log) @@ -547,7 +545,7 @@ func newUsageTestEnv( T: t, Scheme: scheme, K8sClient: k8sClient, - NovaClient: novaClient, + DBClient: dbClient, FlavorGroups: flavorGroups, HTTPServer: httpServer, API: api, diff --git a/internal/scheduling/reservations/commitments/controller.go b/internal/scheduling/reservations/commitments/controller.go index a3aef918c..79d0dbcb1 100644 --- a/internal/scheduling/reservations/commitments/controller.go +++ b/internal/scheduling/reservations/commitments/controller.go @@ -48,6 +48,9 @@ type CommitmentReservationController struct { SchedulerClient *reservations.SchedulerClient // NovaClient for direct Nova API calls (real-time VM status). NovaClient schedulingnova.NovaClient + // UsageAPI is the HTTP API that needs the DB client for report-usage. + // Set this to wire up the DB after Init connects to Postgres. + UsageAPI *HTTPAPI } // Reconcile is part of the main kubernetes reconciliation loop which aims to @@ -575,6 +578,9 @@ func (r *CommitmentReservationController) Init(ctx context.Context, client clien return fmt.Errorf("failed to initialize database connection: %w", err) } logf.FromContext(ctx).Info("database connection initialized for commitment reservation controller") + if r.UsageAPI != nil { + r.UsageAPI.SetUsageDB(NewDBUsageClient(r.DB)) + } } // Initialize scheduler client diff --git a/internal/scheduling/reservations/commitments/usage.go b/internal/scheduling/reservations/commitments/usage.go index b9f662de4..26340b4e9 100644 --- a/internal/scheduling/reservations/commitments/usage.go +++ b/internal/scheduling/reservations/commitments/usage.go @@ -5,8 +5,10 @@ package commitments import ( "context" + "encoding/json" "fmt" "sort" + "strconv" "time" "github.com/cobaltcore-dev/cortex/api/v1alpha1" @@ -29,26 +31,24 @@ type VMUsageInfo struct { MemoryMB uint64 VCPUs uint64 DiskGB uint64 + VideoRAMMiB *uint64 // optional, from flavor extra_specs hw_video:ram_max_mb AZ string Hypervisor string CreatedAt time.Time - UsageMultiple uint64 // Memory in multiples of smallest flavor in the group - Metadata map[string]string // Server metadata from Nova - Tags []string // Server tags from Nova - OSType string // OS type from OSTypeProber (for billing) + UsageMultiple uint64 // Memory in multiples of smallest flavor in the group } // UsageCalculator computes usage reports for Limes LIQUID API. type UsageCalculator struct { - client client.Client - novaClient UsageNovaClient + client client.Client + usageDB UsageDBClient } // NewUsageCalculator creates a new UsageCalculator instance. -func NewUsageCalculator(client client.Client, novaClient UsageNovaClient) *UsageCalculator { +func NewUsageCalculator(client client.Client, usageDB UsageDBClient) *UsageCalculator { return &UsageCalculator{ - client: client, - novaClient: novaClient, + client: client, + usageDB: usageDB, } } @@ -175,7 +175,7 @@ func (c *UsageCalculator) buildCommitmentCapacityMap( return result, nil } -// getProjectVMs retrieves all VMs for a project from Nova and enriches them with flavor group info. +// getProjectVMs retrieves all VMs for a project from Postgres and enriches them with flavor group info. func (c *UsageCalculator) getProjectVMs( ctx context.Context, log logr.Logger, @@ -184,15 +184,14 @@ func (c *UsageCalculator) getProjectVMs( allAZs []liquid.AvailabilityZone, ) ([]VMUsageInfo, error) { - if c.novaClient == nil { - log.Info("Nova client not configured - returning empty VM list", "projectID", projectID) + if c.usageDB == nil { + log.Info("usage DB client not configured - returning empty VM list", "projectID", projectID) return []VMUsageInfo{}, nil } - // Query VMs from Nova - servers, err := c.novaClient.ListProjectServers(ctx, projectID) + rows, err := c.usageDB.ListProjectVMs(ctx, projectID) if err != nil { - return nil, fmt.Errorf("failed to list servers from Nova: %w", err) + return nil, fmt.Errorf("failed to list VMs from Postgres: %w", err) } // Build flavor name -> flavor group lookup @@ -210,49 +209,56 @@ func (c *UsageCalculator) getProjectVMs( } } - // Convert to VMUsageInfo var vms []VMUsageInfo - for _, server := range servers { + for _, row := range rows { // Parse creation time (Nova returns ISO 8601/RFC3339 format) - createdAt, err := time.Parse(time.RFC3339, server.Created) + createdAt, err := time.Parse(time.RFC3339, row.Created) if err != nil { log.V(1).Info("failed to parse server creation time, using zero time", - "server", server.ID, "created", server.Created, "error", err.Error()) + "server", row.ID, "created", row.Created, "error", err.Error()) createdAt = time.Time{} } // Determine flavor group - flavorGroup := flavorToGroup[server.FlavorName] + flavorGroup := flavorToGroup[row.FlavorName] // Calculate usage multiple (memory in units of smallest flavor) - // Use floor division (truncate) - actual consumption, not billing var usageMultiple uint64 - if smallestMem := flavorToSmallestMemory[server.FlavorName]; smallestMem > 0 { - usageMultiple = server.FlavorRAM / smallestMem // Floor division (truncate) + if smallestMem := flavorToSmallestMemory[row.FlavorName]; smallestMem > 0 { + usageMultiple = row.FlavorRAM / smallestMem } - // Normalize AZ - empty or unknown AZs become "unknown" (consistent with limes liquid-nova) - normalizedAZ := liquid.NormalizeAZ(server.AvailabilityZone, allAZs) + // Normalize AZ + normalizedAZ := liquid.NormalizeAZ(row.AZ, allAZs) + + // Parse video RAM from flavor extra_specs + var videoRAMMiB *uint64 + if row.FlavorExtras != "" { + var extraSpecs map[string]string + if err := json.Unmarshal([]byte(row.FlavorExtras), &extraSpecs); err == nil { + if val, ok := extraSpecs["hw_video:ram_max_mb"]; ok { + if parsed, err := strconv.ParseUint(val, 10, 64); err == nil { + videoRAMMiB = &parsed + } + } + } + } - vm := VMUsageInfo{ - UUID: server.ID, - Name: server.Name, - FlavorName: server.FlavorName, + vms = append(vms, VMUsageInfo{ + UUID: row.ID, + Name: row.Name, + FlavorName: row.FlavorName, FlavorGroup: flavorGroup, - Status: server.Status, - MemoryMB: server.FlavorRAM, - VCPUs: server.FlavorVCPUs, - DiskGB: server.FlavorDisk, + Status: row.Status, + MemoryMB: row.FlavorRAM, + VCPUs: row.FlavorVCPUs, + DiskGB: row.FlavorDisk, + VideoRAMMiB: videoRAMMiB, AZ: string(normalizedAZ), - Hypervisor: server.Hypervisor, + Hypervisor: row.Hypervisor, CreatedAt: createdAt, UsageMultiple: usageMultiple, - Metadata: server.Metadata, - Tags: server.Tags, - OSType: server.OSType, - } - - vms = append(vms, vm) + }) } return vms, nil @@ -488,30 +494,22 @@ func (c *UsageCalculator) buildUsageResponse( // buildVMAttributes creates the attributes map for a VM subresource. // Follows the liquid-nova format with nested flavor structure. func buildVMAttributes(vm VMUsageInfo, commitmentID string) map[string]any { - // Build metadata map (never nil for JSON) - metadata := vm.Metadata - if metadata == nil { - metadata = map[string]string{} + flavor := map[string]any{ + "name": vm.FlavorName, + "vcpu": vm.VCPUs, + "ram_mib": vm.MemoryMB, + "disk_gib": vm.DiskGB, } - - // Build tags slice (never nil for JSON) - tags := vm.Tags - if tags == nil { - tags = []string{} + if vm.VideoRAMMiB != nil { + flavor["video_ram_mib"] = *vm.VideoRAMMiB } result := map[string]any{ "status": vm.Status, - "metadata": metadata, - "tags": tags, - "flavor": map[string]any{ - "name": vm.FlavorName, - "vcpu": vm.VCPUs, - "ram_mib": vm.MemoryMB, - "disk_gib": vm.DiskGB, - // video_ram_mib omitted when nil - }, - "os_type": vm.OSType, + "metadata": map[string]string{}, + "tags": []string{}, + "flavor": flavor, + "os_type": "", } // Add commitment_id - nil for PAYG, string for committed diff --git a/internal/scheduling/reservations/commitments/usage_db.go b/internal/scheduling/reservations/commitments/usage_db.go new file mode 100644 index 000000000..57babaa7d --- /dev/null +++ b/internal/scheduling/reservations/commitments/usage_db.go @@ -0,0 +1,65 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package commitments + +import ( + "context" + "fmt" + + "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources/plugins/openstack/nova" + "github.com/cobaltcore-dev/cortex/internal/knowledge/db" +) + +// dbUsageClient implements UsageDBClient using a *db.DB directly. +type dbUsageClient struct { + db *db.DB +} + +// NewDBUsageClient creates a UsageDBClient backed by the given database connection. +func NewDBUsageClient(database *db.DB) UsageDBClient { + return &dbUsageClient{db: database} +} + +// vmQueryRow is the scan target for the server+flavor JOIN query. +type vmQueryRow struct { + ID string `db:"id"` + Name string `db:"name"` + Status string `db:"status"` + Created string `db:"created"` + AZ string `db:"az"` + Hypervisor string `db:"hypervisor"` + FlavorName string `db:"flavor_name"` + FlavorRAM uint64 `db:"flavor_ram"` + FlavorVCPUs uint64 `db:"flavor_vcpus"` + FlavorDisk uint64 `db:"flavor_disk"` + FlavorExtras string `db:"flavor_extras"` +} + +// ListProjectVMs returns all VMs for a project joined with their flavor data from Postgres. +func (c *dbUsageClient) ListProjectVMs(_ context.Context, projectID string) ([]VMRow, error) { + query := ` + SELECT + s.id, s.name, s.status, s.created, + s.os_ext_az_availability_zone AS az, + s.os_ext_srv_attr_hypervisor_hostname AS hypervisor, + s.flavor_name, + COALESCE(f.ram, 0) AS flavor_ram, + COALESCE(f.vcpus, 0) AS flavor_vcpus, + COALESCE(f.disk, 0) AS flavor_disk, + COALESCE(f.extra_specs, '') AS flavor_extras + FROM ` + nova.Server{}.TableName() + ` s + LEFT JOIN ` + nova.Flavor{}.TableName() + ` f ON f.name = s.flavor_name + WHERE s.tenant_id = $1` + + var rows []vmQueryRow + if _, err := c.db.Select(&rows, query, projectID); err != nil { + return nil, fmt.Errorf("failed to query VMs for project %s: %w", projectID, err) + } + + result := make([]VMRow, len(rows)) + for i, r := range rows { + result[i] = VMRow(r) + } + return result, nil +} diff --git a/internal/scheduling/reservations/commitments/usage_test.go b/internal/scheduling/reservations/commitments/usage_test.go index 7f9937382..c2c4a6fd8 100644 --- a/internal/scheduling/reservations/commitments/usage_test.go +++ b/internal/scheduling/reservations/commitments/usage_test.go @@ -13,7 +13,6 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" - "github.com/cobaltcore-dev/cortex/internal/scheduling/nova" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "github.com/sapcc/go-api-declarations/liquid" "k8s.io/apimachinery/pkg/api/resource" @@ -41,7 +40,7 @@ func TestUsageCalculator_CalculateUsage(t *testing.T) { tests := []struct { name string projectID string - vms []nova.ServerDetail + vms []VMRow reservations []*v1alpha1.Reservation allAZs []liquid.AvailabilityZone expectedUsage map[string]uint64 // resourceName -> usage @@ -49,7 +48,7 @@ func TestUsageCalculator_CalculateUsage(t *testing.T) { { name: "empty project", projectID: "project-empty", - vms: []nova.ServerDetail{}, + vms: []VMRow{}, reservations: []*v1alpha1.Reservation{}, allAZs: []liquid.AvailabilityZone{"az-a"}, expectedUsage: map[string]uint64{ @@ -59,10 +58,10 @@ func TestUsageCalculator_CalculateUsage(t *testing.T) { { name: "single VM with commitment", projectID: "project-A", - vms: []nova.ServerDetail{ + vms: []VMRow{ { ID: "vm-001", Name: "vm-001", Status: "ACTIVE", - TenantID: "project-A", AvailabilityZone: "az-a", + AZ: "az-a", Created: baseTime.Format(time.RFC3339), FlavorName: "m1.large", FlavorRAM: 4096, FlavorVCPUs: 16, }, @@ -81,10 +80,10 @@ func TestUsageCalculator_CalculateUsage(t *testing.T) { { name: "VM without matching commitment - PAYG", projectID: "project-B", - vms: []nova.ServerDetail{ + vms: []VMRow{ { ID: "vm-002", Name: "vm-002", Status: "ACTIVE", - TenantID: "project-B", AvailabilityZone: "az-a", + AZ: "az-a", Created: baseTime.Format(time.RFC3339), FlavorName: "m1.large", FlavorRAM: 4096, FlavorVCPUs: 16, }, @@ -122,14 +121,14 @@ func TestUsageCalculator_CalculateUsage(t *testing.T) { Build() // Setup mock Nova client - novaClient := &mockUsageNovaClient{ - servers: map[string][]nova.ServerDetail{ + dbClient := &mockUsageDBClient{ + rows: map[string][]VMRow{ tt.projectID: tt.vms, }, } // Create calculator and run - calc := NewUsageCalculator(k8sClient, novaClient) + calc := NewUsageCalculator(k8sClient, dbClient) logger := log.FromContext(ctx) report, err := calc.CalculateUsage(ctx, logger, tt.projectID, tt.allAZs) if err != nil { @@ -331,8 +330,6 @@ func TestBuildVMAttributes(t *testing.T) { MemoryMB: 4096, VCPUs: 16, DiskGB: 100, - Metadata: map[string]string{"env": "prod"}, - Tags: []string{"important"}, } t.Run("with commitment", func(t *testing.T) { @@ -343,20 +340,16 @@ func TestBuildVMAttributes(t *testing.T) { t.Errorf("status = %v, expected ACTIVE", attrs["status"]) } - // Metadata at top level + // Metadata always empty map (tags/metadata not available from Postgres cache) metadata, ok := attrs["metadata"].(map[string]string) - if !ok { - t.Errorf("metadata is not map[string]string: %T", attrs["metadata"]) - } else if metadata["env"] != "prod" { - t.Errorf("metadata[env] = %v, expected prod", metadata["env"]) + if !ok || metadata == nil { + t.Errorf("metadata should be empty map, got %T: %v", attrs["metadata"], attrs["metadata"]) } - // Tags at top level + // Tags always empty slice tags, ok := attrs["tags"].([]string) - if !ok { - t.Errorf("tags is not []string: %T", attrs["tags"]) - } else if len(tags) != 1 || tags[0] != "important" { - t.Errorf("tags = %v, expected [important]", tags) + if !ok || tags == nil { + t.Errorf("tags should be empty slice, got %T: %v", attrs["tags"], attrs["tags"]) } // Flavor is now nested @@ -397,19 +390,8 @@ func TestBuildVMAttributes(t *testing.T) { } }) - t.Run("with nil metadata and tags", func(t *testing.T) { - vmEmpty := VMUsageInfo{ - UUID: "vm-empty", - Name: "empty-vm", - FlavorName: "m1.small", - Status: "ACTIVE", - MemoryMB: 1024, - VCPUs: 2, - DiskGB: 10, - Metadata: nil, - Tags: nil, - } - attrs := buildVMAttributes(vmEmpty, "") + t.Run("empty metadata and tags always returned", func(t *testing.T) { + attrs := buildVMAttributes(vm, "") // Should have empty map and slice, not nil (for JSON serialization) metadata, ok := attrs["metadata"].(map[string]string) @@ -494,7 +476,7 @@ func TestUsageCalculator_ExpiredAndFutureCommitments(t *testing.T) { tests := []struct { name string projectID string - vms []nova.ServerDetail + vms []VMRow reservations []*v1alpha1.Reservation allAZs []liquid.AvailabilityZone expectedActiveCommitment string // non-empty if VM should be assigned to a commitment @@ -502,10 +484,10 @@ func TestUsageCalculator_ExpiredAndFutureCommitments(t *testing.T) { { name: "active commitment - within time range", projectID: "project-A", - vms: []nova.ServerDetail{ + vms: []VMRow{ { ID: "vm-001", Name: "vm-001", Status: "ACTIVE", - TenantID: "project-A", AvailabilityZone: "az-a", + AZ: "az-a", Created: baseTime.Format(time.RFC3339), FlavorName: "m1.large", FlavorRAM: 4096, FlavorVCPUs: 16, }, @@ -526,10 +508,10 @@ func TestUsageCalculator_ExpiredAndFutureCommitments(t *testing.T) { { name: "expired commitment - should be ignored (VM goes to PAYG)", projectID: "project-A", - vms: []nova.ServerDetail{ + vms: []VMRow{ { ID: "vm-001", Name: "vm-001", Status: "ACTIVE", - TenantID: "project-A", AvailabilityZone: "az-a", + AZ: "az-a", Created: baseTime.Format(time.RFC3339), FlavorName: "m1.large", FlavorRAM: 4096, FlavorVCPUs: 16, }, @@ -547,10 +529,10 @@ func TestUsageCalculator_ExpiredAndFutureCommitments(t *testing.T) { { name: "future commitment - should be ignored (VM goes to PAYG)", projectID: "project-A", - vms: []nova.ServerDetail{ + vms: []VMRow{ { ID: "vm-001", Name: "vm-001", Status: "ACTIVE", - TenantID: "project-A", AvailabilityZone: "az-a", + AZ: "az-a", Created: baseTime.Format(time.RFC3339), FlavorName: "m1.large", FlavorRAM: 4096, FlavorVCPUs: 16, }, @@ -568,10 +550,10 @@ func TestUsageCalculator_ExpiredAndFutureCommitments(t *testing.T) { { name: "mixed - only active commitment is used", projectID: "project-A", - vms: []nova.ServerDetail{ + vms: []VMRow{ { ID: "vm-001", Name: "vm-001", Status: "ACTIVE", - TenantID: "project-A", AvailabilityZone: "az-a", + AZ: "az-a", Created: baseTime.Format(time.RFC3339), FlavorName: "m1.large", FlavorRAM: 4096, FlavorVCPUs: 16, }, @@ -623,13 +605,13 @@ func TestUsageCalculator_ExpiredAndFutureCommitments(t *testing.T) { WithObjects(objects...). Build() - novaClient := &mockUsageNovaClient{ - servers: map[string][]nova.ServerDetail{ + dbClient := &mockUsageDBClient{ + rows: map[string][]VMRow{ tt.projectID: tt.vms, }, } - calc := NewUsageCalculator(k8sClient, novaClient) + calc := NewUsageCalculator(k8sClient, dbClient) logger := log.FromContext(ctx) report, err := calc.CalculateUsage(ctx, logger, tt.projectID, tt.allAZs) if err != nil { @@ -692,17 +674,17 @@ func TestUsageMultipleCalculation_FloorDivision(t *testing.T) { tests := []struct { name string - vms []nova.ServerDetail + vms []VMRow expectedRAM uint64 // Expected RAM usage in units expectedCores uint64 // Expected cores usage expectedInstances uint64 }{ { name: "single smallest flavor - 1 unit", - vms: []nova.ServerDetail{ + vms: []VMRow{ { ID: "vm-001", Name: "vm-001", Status: "ACTIVE", - TenantID: "project-A", AvailabilityZone: "az-a", + AZ: "az-a", Created: baseTime.Format(time.RFC3339), FlavorName: "g_k_c1_m2_v2", FlavorRAM: 2032, FlavorVCPUs: 1, }, @@ -713,10 +695,10 @@ func TestUsageMultipleCalculation_FloorDivision(t *testing.T) { }, { name: "2x flavor with overhead - floor(4080/2032) = 2 units, not 3", - vms: []nova.ServerDetail{ + vms: []VMRow{ { ID: "vm-001", Name: "vm-001", Status: "ACTIVE", - TenantID: "project-A", AvailabilityZone: "az-a", + AZ: "az-a", Created: baseTime.Format(time.RFC3339), FlavorName: "g_k_c2_m4_v2", FlavorRAM: 4080, FlavorVCPUs: 2, }, @@ -727,28 +709,28 @@ func TestUsageMultipleCalculation_FloorDivision(t *testing.T) { }, { name: "multiple VMs - RAM units should match cores for fixed ratio", - vms: []nova.ServerDetail{ + vms: []VMRow{ { ID: "vm-001", Name: "vm-001", Status: "ACTIVE", - TenantID: "project-A", AvailabilityZone: "az-a", + AZ: "az-a", Created: baseTime.Format(time.RFC3339), FlavorName: "g_k_c1_m2_v2", FlavorRAM: 2032, FlavorVCPUs: 1, }, { ID: "vm-002", Name: "vm-002", Status: "ACTIVE", - TenantID: "project-A", AvailabilityZone: "az-a", + AZ: "az-a", Created: baseTime.Add(time.Second).Format(time.RFC3339), FlavorName: "g_k_c2_m4_v2", FlavorRAM: 4080, FlavorVCPUs: 2, }, { ID: "vm-003", Name: "vm-003", Status: "ACTIVE", - TenantID: "project-A", AvailabilityZone: "az-a", + AZ: "az-a", Created: baseTime.Add(2 * time.Second).Format(time.RFC3339), FlavorName: "g_k_c4_m16_v2", FlavorRAM: 16368, FlavorVCPUs: 4, }, { ID: "vm-004", Name: "vm-004", Status: "ACTIVE", - TenantID: "project-A", AvailabilityZone: "az-a", + AZ: "az-a", Created: baseTime.Add(3 * time.Second).Format(time.RFC3339), FlavorName: "g_k_c16_m32_v2", FlavorRAM: 32752, FlavorVCPUs: 16, }, @@ -787,13 +769,13 @@ func TestUsageMultipleCalculation_FloorDivision(t *testing.T) { WithObjects(objects...). Build() - novaClient := &mockUsageNovaClient{ - servers: map[string][]nova.ServerDetail{ + dbClient := &mockUsageDBClient{ + rows: map[string][]VMRow{ "project-A": tt.vms, }, } - calc := NewUsageCalculator(k8sClient, novaClient) + calc := NewUsageCalculator(k8sClient, dbClient) logger := log.FromContext(ctx) report, err := calc.CalculateUsage(ctx, logger, "project-A", []liquid.AvailabilityZone{"az-a"}) if err != nil { From 3090a58537c5aba69cdc051cdfd36c133d0fde69 Mon Sep 17 00:00:00 2001 From: mblos Date: Mon, 13 Apr 2026 16:18:03 +0200 Subject: [PATCH 2/2] use PostgresReader --- cmd/manager/main.go | 7 +++---- .../reservations/commitments/api.go | 19 ++++++++++++------- .../reservations/commitments/controller.go | 6 ------ .../reservations/commitments/usage.go | 4 ++-- .../reservations/commitments/usage_db.go | 16 ++++++++-------- 5 files changed, 25 insertions(+), 27 deletions(-) diff --git a/cmd/manager/main.go b/cmd/manager/main.go index f5f3e1f0f..40b3df54f 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -522,10 +522,9 @@ func main() { commitmentsConfig.ApplyDefaults() if err := (&commitments.CommitmentReservationController{ - Client: multiclusterClient, - Scheme: mgr.GetScheme(), - Conf: commitmentsConfig, - UsageAPI: commitmentsAPI, + Client: multiclusterClient, + Scheme: mgr.GetScheme(), + Conf: commitmentsConfig, }).SetupWithManager(mgr, multiclusterClient); err != nil { setupLog.Error(err, "unable to create controller", "controller", "CommitmentReservation") os.Exit(1) diff --git a/internal/scheduling/reservations/commitments/api.go b/internal/scheduling/reservations/commitments/api.go index ff32a413f..3c43d0a67 100644 --- a/internal/scheduling/reservations/commitments/api.go +++ b/internal/scheduling/reservations/commitments/api.go @@ -9,6 +9,7 @@ import ( "strings" "sync" + "github.com/cobaltcore-dev/cortex/internal/scheduling/external" "github.com/go-logr/logr" "github.com/prometheus/client_golang/prometheus" "sigs.k8s.io/controller-runtime/pkg/client" @@ -52,9 +53,18 @@ func NewAPI(client client.Client) *HTTPAPI { return NewAPIWithConfig(client, DefaultConfig(), nil) } -func NewAPIWithConfig(client client.Client, config Config, usageDB UsageDBClient) *HTTPAPI { +// NewAPIWithConfig creates an HTTPAPI. If usageDB is nil and config.DatabaseSecretRef +// is set, a lazy-connecting PostgresReader-backed client is created automatically. +func NewAPIWithConfig(k8sClient client.Client, config Config, usageDB UsageDBClient) *HTTPAPI { + if usageDB == nil && config.DatabaseSecretRef != nil { + reader := &external.PostgresReader{ + Client: k8sClient, + DatabaseSecretRef: *config.DatabaseSecretRef, + } + usageDB = NewDBUsageClient(reader) + } return &HTTPAPI{ - client: client, + client: k8sClient, config: config, usageDB: usageDB, monitor: NewChangeCommitmentsAPIMonitor(), @@ -64,11 +74,6 @@ func NewAPIWithConfig(client client.Client, config Config, usageDB UsageDBClient } } -// SetUsageDB sets the UsageDBClient after construction (e.g. once the K8s cache is ready). -func (api *HTTPAPI) SetUsageDB(usageDB UsageDBClient) { - api.usageDB = usageDB -} - func (api *HTTPAPI) Init(mux *http.ServeMux, registry prometheus.Registerer, log logr.Logger) { registry.MustRegister(&api.monitor) registry.MustRegister(&api.usageMonitor) diff --git a/internal/scheduling/reservations/commitments/controller.go b/internal/scheduling/reservations/commitments/controller.go index 79d0dbcb1..a3aef918c 100644 --- a/internal/scheduling/reservations/commitments/controller.go +++ b/internal/scheduling/reservations/commitments/controller.go @@ -48,9 +48,6 @@ type CommitmentReservationController struct { SchedulerClient *reservations.SchedulerClient // NovaClient for direct Nova API calls (real-time VM status). NovaClient schedulingnova.NovaClient - // UsageAPI is the HTTP API that needs the DB client for report-usage. - // Set this to wire up the DB after Init connects to Postgres. - UsageAPI *HTTPAPI } // Reconcile is part of the main kubernetes reconciliation loop which aims to @@ -578,9 +575,6 @@ func (r *CommitmentReservationController) Init(ctx context.Context, client clien return fmt.Errorf("failed to initialize database connection: %w", err) } logf.FromContext(ctx).Info("database connection initialized for commitment reservation controller") - if r.UsageAPI != nil { - r.UsageAPI.SetUsageDB(NewDBUsageClient(r.DB)) - } } // Initialize scheduler client diff --git a/internal/scheduling/reservations/commitments/usage.go b/internal/scheduling/reservations/commitments/usage.go index 26340b4e9..294aefe9b 100644 --- a/internal/scheduling/reservations/commitments/usage.go +++ b/internal/scheduling/reservations/commitments/usage.go @@ -6,6 +6,7 @@ package commitments import ( "context" "encoding/json" + "errors" "fmt" "sort" "strconv" @@ -185,8 +186,7 @@ func (c *UsageCalculator) getProjectVMs( ) ([]VMUsageInfo, error) { if c.usageDB == nil { - log.Info("usage DB client not configured - returning empty VM list", "projectID", projectID) - return []VMUsageInfo{}, nil + return nil, errors.New("usage DB client not configured") } rows, err := c.usageDB.ListProjectVMs(ctx, projectID) diff --git a/internal/scheduling/reservations/commitments/usage_db.go b/internal/scheduling/reservations/commitments/usage_db.go index 57babaa7d..6d411ae5b 100644 --- a/internal/scheduling/reservations/commitments/usage_db.go +++ b/internal/scheduling/reservations/commitments/usage_db.go @@ -8,17 +8,17 @@ import ( "fmt" "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources/plugins/openstack/nova" - "github.com/cobaltcore-dev/cortex/internal/knowledge/db" + "github.com/cobaltcore-dev/cortex/internal/scheduling/external" ) -// dbUsageClient implements UsageDBClient using a *db.DB directly. +// dbUsageClient implements UsageDBClient using a PostgresReader for lazy connection. type dbUsageClient struct { - db *db.DB + reader *external.PostgresReader } -// NewDBUsageClient creates a UsageDBClient backed by the given database connection. -func NewDBUsageClient(database *db.DB) UsageDBClient { - return &dbUsageClient{db: database} +// NewDBUsageClient creates a UsageDBClient backed by the given PostgresReader. +func NewDBUsageClient(reader *external.PostgresReader) UsageDBClient { + return &dbUsageClient{reader: reader} } // vmQueryRow is the scan target for the server+flavor JOIN query. @@ -37,7 +37,7 @@ type vmQueryRow struct { } // ListProjectVMs returns all VMs for a project joined with their flavor data from Postgres. -func (c *dbUsageClient) ListProjectVMs(_ context.Context, projectID string) ([]VMRow, error) { +func (c *dbUsageClient) ListProjectVMs(ctx context.Context, projectID string) ([]VMRow, error) { query := ` SELECT s.id, s.name, s.status, s.created, @@ -53,7 +53,7 @@ func (c *dbUsageClient) ListProjectVMs(_ context.Context, projectID string) ([]V WHERE s.tenant_id = $1` var rows []vmQueryRow - if _, err := c.db.Select(&rows, query, projectID); err != nil { + if err := c.reader.Select(ctx, &rows, query, projectID); err != nil { return nil, fmt.Errorf("failed to query VMs for project %s: %w", projectID, err) }