diff --git a/Taskfile.yaml b/Taskfile.yaml index 95f2670d15..ed982758fe 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -89,6 +89,12 @@ tasks: cmds: - werf build {{ .target }} + generate:kubevirt-rewrite-rules: + desc: "Generate YAML from kubevirt rewrite rules" + cmds: + - go run ./cmd/generate-rules/ > ../../templates/kubevirt/_rewrite_rules_data.tpl + dir: ./images/kube-api-rewriter + dev:format:yaml: desc: "Format non-templated YAML files, e.g. CRDs" cmds: diff --git a/build/components/versions.yml b/build/components/versions.yml index f72a94e45d..f737847471 100644 --- a/build/components/versions.yml +++ b/build/components/versions.yml @@ -3,7 +3,7 @@ firmware: libvirt: v10.9.0 edk2: stable202411 core: - 3p-kubevirt: v1.6.2-v12n.21 + 3p-kubevirt: feat/virt-handler-to-hostnetwork # v1.6.2-v12n.21 3p-containerized-data-importer: v1.60.3-v12n.17 distribution: 2.8.3 package: diff --git a/images/kube-api-rewriter/cmd/generate-rules/main.go b/images/kube-api-rewriter/cmd/generate-rules/main.go new file mode 100644 index 0000000000..ed759d3b31 --- /dev/null +++ b/images/kube-api-rewriter/cmd/generate-rules/main.go @@ -0,0 +1,41 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "os" + + "sigs.k8s.io/yaml" + + "github.com/deckhouse/virtualization/kube-api-rewriter/pkg/kubevirt" +) + +func main() { + data, err := yaml.Marshal(kubevirt.KubevirtRewriteRules) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to marshal rewrite rules: %v\n", err) + os.Exit(1) + } + + fmt.Print("{{- define \"kubevirt.rewrite_rules_data\" -}}\n") + if _, err := os.Stdout.Write(data); err != nil { + fmt.Fprintf(os.Stderr, "failed to write output: %v\n", err) + os.Exit(1) + } + fmt.Print("{{- end -}}\n") +} diff --git a/images/kube-api-rewriter/go.mod b/images/kube-api-rewriter/go.mod index c4df163665..590de026cb 100644 --- a/images/kube-api-rewriter/go.mod +++ b/images/kube-api-rewriter/go.mod @@ -11,12 +11,17 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/emicklei/go-restful/v3 v3.12.2 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-openapi/jsonpointer v0.21.1 // indirect + github.com/go-openapi/jsonreference v0.21.0 // indirect github.com/go-openapi/swag v0.23.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/gnostic-models v0.6.9 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/josephburnett/jd v1.9.2 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -25,6 +30,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.23.0 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.65.0 // indirect @@ -43,13 +49,16 @@ require ( golang.org/x/text v0.27.0 // indirect golang.org/x/time v0.12.0 // indirect google.golang.org/protobuf v1.36.6 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/api v0.33.3 // indirect k8s.io/apimachinery v0.33.3 // indirect + k8s.io/apiserver v0.33.3 // indirect k8s.io/client-go v0.33.3 // indirect k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect sigs.k8s.io/controller-runtime v0.21.0 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect @@ -64,3 +73,5 @@ replace ( golang.org/x/net => golang.org/x/net v0.40.0 // CVE-2025-22870, CVE-2025-22872 golang.org/x/oauth2 => golang.org/x/oauth2 v0.27.0 // CVE-2025-22868 ) + +replace github.com/deckhouse/kube-api-rewriter => github.com/yaroslavborbat/kube-api-rewriter v0.0.0-20260402203155-ce012e9b14c8 // feat/auth-rbac-middleware diff --git a/images/kube-api-rewriter/go.sum b/images/kube-api-rewriter/go.sum index 5a6965736f..d23e00ca11 100644 --- a/images/kube-api-rewriter/go.sum +++ b/images/kube-api-rewriter/go.sum @@ -6,8 +6,6 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/deckhouse/kube-api-rewriter v0.1.2 h1:FQiVAbj73Sm5MmTvuA73wFM8mHQkJlq9oDlHLNw2Yy8= -github.com/deckhouse/kube-api-rewriter v0.1.2/go.mod h1:tZFw2byvVh4C0D/RxAAgp2x929yTUv9+sN2zZy59hNE= github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= @@ -30,8 +28,8 @@ github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1v github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= -github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= +github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -104,6 +102,8 @@ github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yaroslavborbat/kube-api-rewriter v0.0.0-20260402203155-ce012e9b14c8 h1:2sMKqgWgX9O80bAfJHmgw81EIBpfw4PxUH1uf6vU/d0= +github.com/yaroslavborbat/kube-api-rewriter v0.0.0-20260402203155-ce012e9b14c8/go.mod h1:6xreNakzKpoQ6btk+tViQ1F3QFRksDR7vHGNysoIymQ= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= @@ -190,12 +190,14 @@ k8s.io/api v0.33.3 h1:SRd5t//hhkI1buzxb288fy2xvjubstenEKL9K51KBI8= k8s.io/api v0.33.3/go.mod h1:01Y/iLUjNBM3TAvypct7DIj0M0NIZc+PzAHCIo0CYGE= k8s.io/apimachinery v0.33.3 h1:4ZSrmNa0c/ZpZJhAgRdcsFcZOw1PQU1bALVQ0B3I5LA= k8s.io/apimachinery v0.33.3/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= +k8s.io/apiserver v0.33.3 h1:Wv0hGc+QFdMJB4ZSiHrCgN3zL3QRatu56+rpccKC3J4= +k8s.io/apiserver v0.33.3/go.mod h1:05632ifFEe6TxwjdAIrwINHWE2hLwyADFk5mBsQa15E= k8s.io/client-go v0.33.3 h1:M5AfDnKfYmVJif92ngN532gFqakcGi6RvaOF16efrpA= k8s.io/client-go v0.33.3/go.mod h1:luqKBQggEf3shbxHY4uVENAxrDISLOarxpTKMiUuujg= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b h1:MloQ9/bdJyIu9lb1PzujOPolHyvO06MXG5TUIj2mNAA= -k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts= +k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4= +k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8= k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/controller-runtime v0.21.0 h1:CYfjpEuicjUecRk+KAeyYh+ouUBn4llGyDYytIGcJS8= @@ -207,8 +209,6 @@ sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= sigs.k8s.io/structured-merge-diff/v4 v4.7.0 h1:qPeWmscJcXP0snki5IYF79Z8xrl8ETFxgMd7wez1XkI= sigs.k8s.io/structured-merge-diff/v4 v4.7.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps= -sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= -sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/images/kube-api-rewriter/pkg/kubevirt/kubevirt_rules.go b/images/kube-api-rewriter/pkg/kubevirt/kubevirt_rules.go index cc89f3c928..ce9b8349c0 100644 --- a/images/kube-api-rewriter/pkg/kubevirt/kubevirt_rules.go +++ b/images/kube-api-rewriter/pkg/kubevirt/kubevirt_rules.go @@ -17,7 +17,7 @@ limitations under the License. package kubevirt import ( - . "github.com/deckhouse/kube-api-rewriter/pkg/rewriter" + "github.com/deckhouse/kube-api-rewriter/pkg/rewriter" ) const ( @@ -26,15 +26,15 @@ const ( rootPrefix = "virtualization.deckhouse.io" ) -var KubevirtRewriteRules = &RewriteRules{ +var KubevirtRewriteRules = &rewriter.RewriteRules{ KindPrefix: "InternalVirtualization", // VirtualMachine -> InternalVirtualizationVirtualMachine ResourceTypePrefix: "internalvirtualization", // virtualmachines -> internalvirtualizationvirtualmachines ShortNamePrefix: "intvirt", // kubectl get intvirtvm Categories: []string{"intvirt"}, // kubectl get intvirt to see all KubeVirt and CDI resources. Rules: KubevirtAPIGroupsRules, Webhooks: KubevirtWebhooks, - Labels: MetadataReplace{ - Names: []MetadataReplaceRule{ + Labels: rewriter.MetadataReplace{ + Names: []rewriter.MetadataReplaceRule{ {Original: "cdi.kubevirt.io", Renamed: "cdi." + internalPrefix}, {Original: "kubevirt.io", Renamed: "kubevirt." + internalPrefix}, {Original: "operator.kubevirt.io", Renamed: "operator.kubevirt." + internalPrefix}, @@ -60,7 +60,7 @@ var KubevirtRewriteRules = &RewriteRules{ Renamed: "app.kubernetes.io/managed-by", RenamedValue: "kubevirt-operator-internal-virtualization", }, }, - Prefixes: []MetadataReplaceRule{ + Prefixes: []rewriter.MetadataReplaceRule{ // CDI related labels. {Original: "cdi.kubevirt.io", Renamed: "cdi." + internalPrefix}, {Original: "operator.cdi.kubevirt.io", Renamed: "operator.cdi." + internalPrefix}, @@ -85,8 +85,8 @@ var KubevirtRewriteRules = &RewriteRules{ {Original: "machine-type.node.kubevirt.io", Renamed: "machine-type." + nodePrefix}, }, }, - Annotations: MetadataReplace{ - Prefixes: []MetadataReplaceRule{ + Annotations: rewriter.MetadataReplace{ + Prefixes: []rewriter.MetadataReplaceRule{ // CDI related annotations. {Original: "cdi.kubevirt.io", Renamed: "cdi." + internalPrefix}, {Original: "operator.cdi.kubevirt.io", Renamed: "operator.cdi." + internalPrefix}, @@ -95,14 +95,14 @@ var KubevirtRewriteRules = &RewriteRules{ {Original: "certificates.kubevirt.io", Renamed: "certificates.kubevirt." + internalPrefix}, }, }, - Finalizers: MetadataReplace{ - Prefixes: []MetadataReplaceRule{ + Finalizers: rewriter.MetadataReplace{ + Prefixes: []rewriter.MetadataReplaceRule{ {Original: "kubevirt.io", Renamed: "kubevirt." + internalPrefix}, {Original: "operator.cdi.kubevirt.io", Renamed: "operator.cdi." + internalPrefix}, }, }, - Excludes: []ExcludeRule{ - ExcludeRule{ + Excludes: []rewriter.ExcludeRule{ + rewriter.ExcludeRule{ Kinds: []string{ "PersistentVolumeClaim", "PersistentVolume", @@ -112,7 +112,7 @@ var KubevirtRewriteRules = &RewriteRules{ "app.kubernetes.io/managed-by": "cdi-controller", }, }, - ExcludeRule{ + rewriter.ExcludeRule{ Kinds: []string{ "CDI", }, @@ -125,15 +125,15 @@ var KubevirtRewriteRules = &RewriteRules{ // TODO create generator in golang to produce below rules from Kubevirt and CDI sources so proxy can work with future versions. -var KubevirtAPIGroupsRules = map[string]APIGroupRule{ +var KubevirtAPIGroupsRules = map[string]rewriter.APIGroupRule{ "cdi.kubevirt.io": { - GroupRule: GroupRule{ + GroupRule: rewriter.GroupRule{ Group: "cdi.kubevirt.io", Versions: []string{"v1beta1"}, PreferredVersion: "v1beta1", Renamed: "cdi." + internalPrefix, }, - ResourceRules: map[string]ResourceRule{ + ResourceRules: map[string]rewriter.ResourceRule{ // cdiconfigs.cdi.kubevirt.io "cdiconfigs": { Kind: "CDIConfig", @@ -247,13 +247,13 @@ var KubevirtAPIGroupsRules = map[string]APIGroupRule{ }, }, "forklift.cdi.kubevirt.io": { - GroupRule: GroupRule{ + GroupRule: rewriter.GroupRule{ Group: "forklift.cdi.kubevirt.io", Versions: []string{"v1beta1"}, PreferredVersion: "v1beta1", Renamed: "forklift.cdi." + internalPrefix, }, - ResourceRules: map[string]ResourceRule{ + ResourceRules: map[string]rewriter.ResourceRule{ // openstackvolumepopulators.forklift.cdi.kubevirt.io "openstackvolumepopulators": { Kind: "OpenstackVolumePopulator", @@ -277,13 +277,13 @@ var KubevirtAPIGroupsRules = map[string]APIGroupRule{ }, }, "kubevirt.io": { - GroupRule: GroupRule{ + GroupRule: rewriter.GroupRule{ Group: "kubevirt.io", Versions: []string{"v1", "v1alpha3"}, PreferredVersion: "v1", Renamed: "internal.virtualization.deckhouse.io", }, - ResourceRules: map[string]ResourceRule{ + ResourceRules: map[string]rewriter.ResourceRule{ // kubevirts.kubevirt.io "kubevirts": { Kind: "KubeVirt", @@ -353,13 +353,13 @@ var KubevirtAPIGroupsRules = map[string]APIGroupRule{ }, }, "clone.kubevirt.io": { - GroupRule: GroupRule{ + GroupRule: rewriter.GroupRule{ Group: "clone.kubevirt.io", Versions: []string{"v1alpha1"}, PreferredVersion: "v1alpha1", Renamed: "clone.internal.virtualization.deckhouse.io", }, - ResourceRules: map[string]ResourceRule{ + ResourceRules: map[string]rewriter.ResourceRule{ // virtualmachineclones.clone.kubevirt.io "virtualmachineclones": { Kind: "VirtualMachineClone", @@ -374,13 +374,13 @@ var KubevirtAPIGroupsRules = map[string]APIGroupRule{ }, }, "export.kubevirt.io": { - GroupRule: GroupRule{ + GroupRule: rewriter.GroupRule{ Group: "export.kubevirt.io", Versions: []string{"v1alpha1"}, PreferredVersion: "v1alpha1", Renamed: "export.internal.virtualization.deckhouse.io", }, - ResourceRules: map[string]ResourceRule{ + ResourceRules: map[string]rewriter.ResourceRule{ // virtualmachineexports.export.kubevirt.io "virtualmachineexports": { Kind: "VirtualMachineExport", @@ -395,13 +395,13 @@ var KubevirtAPIGroupsRules = map[string]APIGroupRule{ }, }, "instancetype.kubevirt.io": { - GroupRule: GroupRule{ + GroupRule: rewriter.GroupRule{ Group: "instancetype.kubevirt.io", Versions: []string{"v1alpha1", "v1alpha2"}, PreferredVersion: "v1alpha2", Renamed: "instancetype.internal.virtualization.deckhouse.io", }, - ResourceRules: map[string]ResourceRule{ + ResourceRules: map[string]rewriter.ResourceRule{ // virtualmachineinstancetypes.instancetype.kubevirt.io "virtualmachineinstancetypes": { Kind: "VirtualMachineInstancetype", @@ -449,13 +449,13 @@ var KubevirtAPIGroupsRules = map[string]APIGroupRule{ }, }, "migrations.kubevirt.io": { - GroupRule: GroupRule{ + GroupRule: rewriter.GroupRule{ Group: "migrations.kubevirt.io", Versions: []string{"v1alpha1"}, PreferredVersion: "v1alpha1", Renamed: "migrations.internal.virtualization.deckhouse.io", }, - ResourceRules: map[string]ResourceRule{ + ResourceRules: map[string]rewriter.ResourceRule{ // migrationpolicies.migrations.kubevirt.io "migrationpolicies": { Kind: "MigrationPolicy", @@ -470,13 +470,13 @@ var KubevirtAPIGroupsRules = map[string]APIGroupRule{ }, }, "pool.kubevirt.io": { - GroupRule: GroupRule{ + GroupRule: rewriter.GroupRule{ Group: "pool.kubevirt.io", Versions: []string{"v1alpha1"}, PreferredVersion: "v1alpha1", Renamed: "pool.internal.virtualization.deckhouse.io", }, - ResourceRules: map[string]ResourceRule{ + ResourceRules: map[string]rewriter.ResourceRule{ // virtualmachinepools.pool.kubevirt.io "virtualmachinepools": { Kind: "VirtualMachinePool", @@ -491,13 +491,13 @@ var KubevirtAPIGroupsRules = map[string]APIGroupRule{ }, }, "snapshot.kubevirt.io": { - GroupRule: GroupRule{ + GroupRule: rewriter.GroupRule{ Group: "snapshot.kubevirt.io", Versions: []string{"v1alpha1"}, PreferredVersion: "v1alpha1", Renamed: "snapshot.internal.virtualization.deckhouse.io", }, - ResourceRules: map[string]ResourceRule{ + ResourceRules: map[string]rewriter.ResourceRule{ // virtualmachinerestores.snapshot.kubevirt.io "virtualmachinerestores": { Kind: "VirtualMachineRestore", @@ -535,7 +535,7 @@ var KubevirtAPIGroupsRules = map[string]APIGroupRule{ }, } -var KubevirtWebhooks = map[string]WebhookRule{ +var KubevirtWebhooks = map[string]rewriter.WebhookRule{ // CDI webhooks. // Run this in original CDI installation: // kubectl get validatingwebhookconfiguration,mutatingwebhookconfiguration -l cdi.kubevirt.io -o json | jq '.items[] | .webhooks[] | {"path": .clientConfig.service.path, "group": (.rules[]|.apiGroups|join(",")), "resource": (.rules[]|.resources|join(",")) } | "\""+.path +"\": {\nPath: \"" + .path + "\",\nGroup: \"" + .group + "\",\nResource: \"" + .resource + "\",\n}," ' -r diff --git a/images/virt-artifact/werf.inc.yaml b/images/virt-artifact/werf.inc.yaml index f30560fba6..7b186d51ff 100644 --- a/images/virt-artifact/werf.inc.yaml +++ b/images/virt-artifact/werf.inc.yaml @@ -9,6 +9,7 @@ image: {{ .ModuleNamePrefix }}{{ .ImageName }}-src-artifact final: false fromImage: builder/src +fromCacheVersion: "014" # TODO: DELETE ME secrets: - id: SOURCE_REPO value: {{ $.SOURCE_REPO }} diff --git a/templates/_hostnetwork_ports.tpl b/templates/_hostnetwork_ports.tpl new file mode 100644 index 0000000000..874183edfb --- /dev/null +++ b/templates/_hostnetwork_ports.tpl @@ -0,0 +1,47 @@ +{{- /* +Port constants for DaemonSets running with hostNetwork: true. + +All three DaemonSets — virt-handler, vm-route-forge, virtualization-dra — +run with hostNetwork, so every bound port is exposed on the node's network +interfaces. Ports below are chosen outside the KubeVirt live-migration range +(4135-4199) and must not overlap with other well-known services on cluster nodes. + +Port map: + + virt-handler (kube-api-rewriter runs as its sidecar): + 4135-4199 virt-handler: live-migration tunnels (KubeVirt migration range). + 4100 virt-handler: healthz and Prometheus metrics (--port flag), kube-rbac-proxy implemented natively. + 4101 virt-handler: Console server port (--console-server-port flag). + 4102 kube-api-rewriter sidecar: Prometheus metrics (MONITORING_BIND_ADDRESS), bound to pod IP. + liveness and readiness probes (/proxy/healthz, /proxy/readyz). + 4103 kube-api-rewriter sidecar: pprof (PPROF_BIND_ADDRESS), bound to pod IP, debug mode only. + 4104 kube-api-rewriter sidecar: Kubernetes API proxy (CLIENT_PROXY_PORT), + virt-handler connects here instead of the real API server. + + vm-route-forge: + 4105 vm-route-forge: liveness and readiness probes (HEALTH_PROBE_BIND_ADDRESS). + 4106 vm-route-forge: pprof (PPROF_BIND_ADDRESS), debug mode only. + + virtualization-dra: + 4107 virtualization-dra: gRPC liveness and readiness probes. + 4280 virtualization-dra: USB/IP daemon (--usbipd-port flag). +*/ -}} + +{{- /* virt-handler */ -}} +{{- define "virt_handler.migration_port_first" -}}4135{{- end -}} +{{- define "virt_handler.migration_port_last" -}}4199{{- end -}} + +{{- define "virt_handler.port" -}}4100{{- end -}} +{{- define "virt_handler.console_server_port" -}}4101{{- end -}} +{{- define "virt_handler.rewriter_healthz_port" -}}4102{{- end -}} +{{- define "virt_handler.rewriter_monitoring_port" -}}4102{{- end -}} +{{- define "virt_handler.rewriter_pprof_port" -}}4103{{- end -}} +{{- define "virt_handler.rewriter_proxy_port" -}}4104{{- end -}} + +{{- /* vm-route-forge */ -}} +{{- define "vm_route_forge.health_port" -}}4105{{- end -}} +{{- define "vm_route_forge.pprof_port" -}}4106{{- end -}} + +{{- /* virtualization-dra */ -}} +{{- define "virtualization_dra.health_port" -}}4107{{- end -}} +{{- define "virtualization_dra.usbipd_port" -}}4280{{- end -}} diff --git a/templates/kube-api-rewriter/_customize_patch_helpers.tpl b/templates/kube-api-rewriter/_customize_patch_helpers.tpl index 72b1d18bbd..ad361d8d3e 100644 --- a/templates/kube-api-rewriter/_customize_patch_helpers.tpl +++ b/templates/kube-api-rewriter/_customize_patch_helpers.tpl @@ -30,7 +30,7 @@ spec: {{- include "kube_api_rewriter.sidecar_container" (tuple $ctx $settings) | nindent 6 }} - name: {{ $mainContainerName }} env: - {{- include "kube_api_rewriter.kubeconfig_env" . | nindent 8 }} + {{- include "kube_api_rewriter.kubeconfig_env" (tuple $ctx $settings) | nindent 8 }} volumeMounts: {{- include "kube_api_rewriter.kubeconfig_volume_mount" . | nindent 8 }} {{- end -}} diff --git a/templates/kube-api-rewriter/_settings.tpl b/templates/kube-api-rewriter/_settings.tpl index 8f54135195..f9703033a3 100644 --- a/templates/kube-api-rewriter/_settings.tpl +++ b/templates/kube-api-rewriter/_settings.tpl @@ -7,13 +7,11 @@ {{- define "kube_api_rewriter.pprof_port" -}}8129{{- end -}} +{{- define "kube_api_rewriter.client_proxy_port" -}}23915{{- end -}} + {{- define "kube_api_rewriter.env" -}} - name: LOG_LEVEL value: {{ include "moduleLogLevel" . }} -{{- if eq (include "moduleLogLevel" .) "debug" }} -- name: PPROF_BIND_ADDRESS - value: ":{{ include "kube_api_rewriter.pprof_port" . }}" -{{- end }} {{- end -}} {{- define "kube_api_rewriter.resources" -}} diff --git a/templates/kube-api-rewriter/_sidecar_helpers.tpl b/templates/kube-api-rewriter/_sidecar_helpers.tpl index 2ae379c146..59013d0c1e 100644 --- a/templates/kube-api-rewriter/_sidecar_helpers.tpl +++ b/templates/kube-api-rewriter/_sidecar_helpers.tpl @@ -1,98 +1,17 @@ -{{- /* Helpers to add kube-api-rewriter sidecar container to a pod. - -To connect to kube-api-rewriter main controller should has KUBECONFIG env, -volumeMount with kubeconfig, and Pod should has volume with kubeconfig ConfigMap. - -These settings are provided by helpers: - -- kube_api_rewriter.kubeconfig_env defines KUBECONFIG env with file from the - mounted ConfigMap. -- kube_api_rewriter.kubeconfig_volume_mount defines volumeMount for kubeconfig ConfigMap. -- kube_api_rewriter.kubeconfig_volume defines volume with kubeconfig ConfigMap. - -Kube-api-rewriter sidecar should be the first container in the Pod, to -main controller not fail on start. - -Kube-api-rewriter sidecar works in 2 modes: without webhook or with webhook rewriting. - -Sidecar without webhook is the simplest one: - -spec: - template: - spec: - containers: - {{ include "kube_api_rewriter.sidecar_container" . | nindent 8 }} - - name: main-controller - ... - env: - {{- include "kube_api_rewriter.kubeconfig_env" . | nindent 12 }} - ... - volumeMounts: - {{- include "kube_api_rewriter.kubeconfig_volume_mount" . | nindent 12 }} - ... - volumes: - {{- include "kube_api_rewriter.kubeconfig_volume" | nindent 8 }} - ... - - -Webhook mode requires additional settings: - -- WEBHOOK_ADDRESS - address of the webhook in the main controller -- WEBHOOK_CERT_FILE - path to the webhook certificate file. -- WEBHOOK_KEY_FILE - path to the webhook key file. -- webhookCertsVolumeName - name of the Pod volume with webhook certificates. -- webhookCertsMountPath - path to mount the webhook certificates. - -The assumption here is that main controller has a webhook server and -certificates are already mounted in the Pod, so kube-api-rewriter -can use certificates from that volume to impersonate the webhook server. - -Example of adding kube-api-rewriter to the Deployment: - -spec: - template: - spec: - containers: - {{- $rewriterSettings := dict }} - {{- $_ := set $rewriterSettings "WEBHOOK_ADDRESS" "https://127.0.0.1:6443" }} - {{- $_ := set $rewriterSettings "WEBHOOK_CERT_FILE" "/etc/webhook-certificates/tls.crt" }} - {{- $_ := set $rewriterSettings "WEBHOOK_KEY_FILE" "/etc/webhook-certificates/tls.key" }} - {{- $_ := set $rewriterSettings "webhookCertsVolumeName" "webhook-certs" }} - {{- $_ := set $rewriterSettings "webhookCertsMountPath" "/etc/webhook-certificates" }} - {{- include "kube_api_rewriter.sidecar_container" (tuple . $rewriterSettings) | nindent 6 }} - - name: main-controller - ... - env: - {{- include "kube_api_rewriter.kubeconfig_env" . | nindent 12 }} - ... - ports: - - containerPort: 6443 # Goes to the WEBHOOK_ADDRESS - name: webhooks - protocol: TCP - volumeMounts: - {{- include "kube_api_rewriter.kubeconfig_volume_mount" . | nindent 12 }} - - name: webhook-certs - mountPath: /etc/webhook-certificates # Goes to the webhookCertsMountPath - readOnly: true - ... - volumes: - {{- include "kube_api_rewriter.kubeconfig_volume" | nindent 8 }} - - name: webhook-certs # Name of the existing volume goes to the webhookCertsVolumeName. - secret: - optional: true - secretName: webhook-certs - ... - - */ -}} - {{- define "kube_api_rewriter.image" -}} {{- include "helm_lib_module_image" (list . "kubeApiRewriter") | toJson -}} {{- end -}} - {{- define "kube_api_rewriter.kubeconfig_env" -}} +{{- $settings := dict -}} +{{- if (kindIs "slice" .) -}} +{{- if ge (len .) 2 -}} +{{- $settings = index . 1 -}} +{{- end -}} +{{- end -}} +{{- $kubeconfigFilename := $settings.kubeconfigFilename | default "kube-api-rewriter.kubeconfig" -}} - name: KUBECONFIG - value: /kubeconfig.local/kube-api-rewriter.kubeconfig + value: /kubeconfig.local/{{ $kubeconfigFilename }} {{- end }} {{- define "kube_api_rewriter.kubeconfig_volume" -}} @@ -107,7 +26,6 @@ spec: mountPath: /kubeconfig.local {{- end }} - {{- define "kube_api_rewriter.webhook_volume_mount" -}} {{- $volumeName := index . 0 -}} {{- $mountPath := index . 1 -}} @@ -122,16 +40,12 @@ spec: protocol: TCP {{- end }} -{{- /* Container port for the pprof server */ -}} {{- define "kube_api_rewriter.pprof_container_port" -}} - containerPort: {{ include "kube_api_rewriter.pprof_port" . }} name: pprof protocol: TCP {{- end }} -{{- /* Sidecar container spec with kube-api-rewriter */ -}} -{{- /* Usage without the webhook proxy: {{ include kube_api_rewriter.sidecar_container . }} */ -}} -{{- /* Usage with the webhook: {{ include kube_api_rewriter.sidecar_container (tuple . $webhookSettings) }} */ -}} {{- define "kube_api_rewriter.sidecar_container" -}} {{- $ctx := . -}} {{- $settings := dict -}} @@ -142,6 +56,15 @@ spec: {{- end -}} {{- end -}} {{- $isWebhook := hasKey $settings "WEBHOOK_ADDRESS" -}} + {{- $injectPodIP := $settings.injectPodIP | default false -}} + {{- $healthzPort := $settings.healthzPort | default 8082 -}} + {{- $healthzPath := $settings.healthzPath | default "/proxy/healthz" -}} + {{- $readyzPath := $settings.readyzPath | default "/proxy/readyz" -}} + {{- $clientProxyPort := $settings.clientProxyPort | default (include "kube_api_rewriter.client_proxy_port" $ctx | int) -}} + {{- $monitoringBindAddress := $settings.monitoringBindAddress | default "127.0.0.1:9090" -}} + {{- $pprofBindAddress := $settings.pprofBindAddress | default (printf ":%s" (include "kube_api_rewriter.pprof_port" $ctx)) -}} + {{- $pprofPort := last (splitList ":" $pprofBindAddress) | int -}} + {{- $probeScheme := $settings.probeScheme | default "HTTPS" -}} - name: {{ include "kube_api_rewriter.sidecar_name" $ctx }} image: {{ include "kube_api_rewriter.image" $ctx }} imagePullPolicy: IfNotPresent @@ -154,8 +77,24 @@ spec: - name: WEBHOOK_KEY_FILE value: "{{ $settings.WEBHOOK_KEY_FILE }}" {{- end }} + {{- if $injectPodIP }} + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + {{- end }} + - name: CLIENT_PROXY_PORT + value: "{{ $clientProxyPort }}" - name: MONITORING_BIND_ADDRESS - value: "127.0.0.1:9090" + value: "{{ $monitoringBindAddress }}" + {{- if $settings.monitoringAuth }} + - name: MONITORING_AUTH + value: {{ $settings.monitoringAuth | toJson | quote }} + {{- end }} + {{- if eq (include "moduleLogLevel" $ctx) "debug" }} + - name: PPROF_BIND_ADDRESS + value: "{{ $pprofBindAddress }}" + {{- end }} {{- include "kube_api_rewriter.env" $ctx | nindent 4 }} resources: requests: @@ -173,15 +112,15 @@ spec: type: RuntimeDefault livenessProbe: httpGet: - path: /proxy/healthz - port: 8082 - scheme: HTTPS + path: {{ $healthzPath }} + port: {{ $healthzPort }} + scheme: {{ $probeScheme }} initialDelaySeconds: 10 readinessProbe: httpGet: - path: /proxy/readyz - port: 8082 - scheme: HTTPS + path: {{ $readyzPath }} + port: {{ $healthzPort }} + scheme: {{ $probeScheme }} initialDelaySeconds: 10 terminationMessagePath: /dev/termination-log terminationMessagePolicy: File @@ -191,9 +130,13 @@ spec: {{- end }} ports: {{- if eq (include "moduleLogLevel" $ctx) "debug" }} - {{- include "kube_api_rewriter.pprof_container_port" . | nindent 4 }} + - containerPort: {{ $pprofPort }} + name: pprof + protocol: TCP {{- end }} - {{- if $isWebhook -}} - {{- include "kube_api_rewriter.webhook_container_port" .| nindent 4 }} + {{- if $isWebhook }} + - containerPort: {{ include "kube_api_rewriter.webhook_port" $ctx }} + name: {{ include "kube_api_rewriter.webhook_port_name" $ctx }} + protocol: TCP {{- end -}} {{- end -}} diff --git a/templates/kube-api-rewriter/cm-kubeconfig-local.yaml b/templates/kube-api-rewriter/cm-kubeconfig-local.yaml index 966a348e5b..bbdbb380d6 100644 --- a/templates/kube-api-rewriter/cm-kubeconfig-local.yaml +++ b/templates/kube-api-rewriter/cm-kubeconfig-local.yaml @@ -18,3 +18,4 @@ data: cluster: kube-api-rewriter name: kube-api-rewriter current-context: kube-api-rewriter + diff --git a/templates/kubevirt/_rewrite_rules_data.tpl b/templates/kubevirt/_rewrite_rules_data.tpl new file mode 100644 index 0000000000..f45227970c --- /dev/null +++ b/templates/kubevirt/_rewrite_rules_data.tpl @@ -0,0 +1,734 @@ +{{- define "kubevirt.rewrite_rules_data" -}} +annotations: + Names: null + Prefixes: + - original: cdi.kubevirt.io + originalValue: "" + renamed: cdi.internal.virtualization.deckhouse.io + renamedValue: "" + - original: operator.cdi.kubevirt.io + originalValue: "" + renamed: operator.cdi.internal.virtualization.deckhouse.io + renamedValue: "" + - original: kubevirt.io + originalValue: "" + renamed: kubevirt.internal.virtualization.deckhouse.io + renamedValue: "" + - original: certificates.kubevirt.io + originalValue: "" + renamed: certificates.kubevirt.internal.virtualization.deckhouse.io + renamedValue: "" +categories: +- intvirt +excludes: +- kinds: + - PersistentVolumeClaim + - PersistentVolume + - Pod + matchLabels: + app.kubernetes.io/managed-by: cdi-controller + matchNames: null +- kinds: + - CDI + matchLabels: null + matchNames: + - cdi +finalizers: + Names: null + Prefixes: + - original: kubevirt.io + originalValue: "" + renamed: kubevirt.internal.virtualization.deckhouse.io + renamedValue: "" + - original: operator.cdi.kubevirt.io + originalValue: "" + renamed: operator.cdi.internal.virtualization.deckhouse.io + renamedValue: "" +kindPrefix: InternalVirtualization +labels: + Names: + - original: cdi.kubevirt.io + originalValue: "" + renamed: cdi.internal.virtualization.deckhouse.io + renamedValue: "" + - original: kubevirt.io + originalValue: "" + renamed: kubevirt.internal.virtualization.deckhouse.io + renamedValue: "" + - original: operator.kubevirt.io + originalValue: "" + renamed: operator.kubevirt.internal.virtualization.deckhouse.io + renamedValue: "" + - original: prometheus.kubevirt.io + originalValue: "" + renamed: prometheus.kubevirt.internal.virtualization.deckhouse.io + renamedValue: "" + - original: prometheus.cdi.kubevirt.io + originalValue: "" + renamed: prometheus.cdi.internal.virtualization.deckhouse.io + renamedValue: "" + - original: node-labeller.kubevirt.io/skip-node + originalValue: "" + renamed: node-labeller.virtualization.deckhouse.io/skip-node + renamedValue: "" + - original: node-labeller.kubevirt.io/obsolete-host-model + originalValue: "" + renamed: node-labeller.internal.virtualization.deckhouse.io/obsolete-host-model + renamedValue: "" + - original: app.kubernetes.io/managed-by + originalValue: cdi-operator + renamed: app.kubernetes.io/managed-by + renamedValue: cdi-operator-internal-virtualization + - original: app.kubernetes.io/managed-by + originalValue: cdi-controller + renamed: app.kubernetes.io/managed-by + renamedValue: cdi-controller-internal-virtualization + - original: app.kubernetes.io/managed-by + originalValue: virt-operator + renamed: app.kubernetes.io/managed-by + renamedValue: virt-operator-internal-virtualization + - original: app.kubernetes.io/managed-by + originalValue: kubevirt-operator + renamed: app.kubernetes.io/managed-by + renamedValue: kubevirt-operator-internal-virtualization + Prefixes: + - original: cdi.kubevirt.io + originalValue: "" + renamed: cdi.internal.virtualization.deckhouse.io + renamedValue: "" + - original: operator.cdi.kubevirt.io + originalValue: "" + renamed: operator.cdi.internal.virtualization.deckhouse.io + renamedValue: "" + - original: prometheus.cdi.kubevirt.io + originalValue: "" + renamed: prometheus.cdi.internal.virtualization.deckhouse.io + renamedValue: "" + - original: upload.cdi.kubevirt.io + originalValue: "" + renamed: upload.cdi.internal.virtualization.deckhouse.io + renamedValue: "" + - original: kubevirt.io + originalValue: "" + renamed: kubevirt.internal.virtualization.deckhouse.io + renamedValue: "" + - original: prometheus.kubevirt.io + originalValue: "" + renamed: prometheus.kubevirt.internal.virtualization.deckhouse.io + renamedValue: "" + - original: operator.kubevirt.io + originalValue: "" + renamed: operator.kubevirt.internal.virtualization.deckhouse.io + renamedValue: "" + - original: vm.kubevirt.io + originalValue: "" + renamed: vm.kubevirt.internal.virtualization.deckhouse.io + renamedValue: "" + - original: cpu-feature.node.kubevirt.io + originalValue: "" + renamed: cpu-feature.node.virtualization.deckhouse.io + renamedValue: "" + - original: cpu-model-migration.node.kubevirt.io + originalValue: "" + renamed: cpu-model-migration.node.virtualization.deckhouse.io + renamedValue: "" + - original: cpu-model.node.kubevirt.io + originalValue: "" + renamed: cpu-model.node.virtualization.deckhouse.io + renamedValue: "" + - original: cpu-timer.node.kubevirt.io + originalValue: "" + renamed: cpu-timer.node.virtualization.deckhouse.io + renamedValue: "" + - original: cpu-vendor.node.kubevirt.io + originalValue: "" + renamed: cpu-vendor.node.virtualization.deckhouse.io + renamedValue: "" + - original: scheduling.node.kubevirt.io + originalValue: "" + renamed: scheduling.node.virtualization.deckhouse.io + renamedValue: "" + - original: host-model-cpu.node.kubevirt.io + originalValue: "" + renamed: host-model-cpu.node.virtualization.deckhouse.io + renamedValue: "" + - original: host-model-required-features.node.kubevirt.io + originalValue: "" + renamed: host-model-required-features.node.virtualization.deckhouse.io + renamedValue: "" + - original: hyperv.node.kubevirt.io + originalValue: "" + renamed: hyperv.node.virtualization.deckhouse.io + renamedValue: "" + - original: machine-type.node.kubevirt.io + originalValue: "" + renamed: machine-type.node.virtualization.deckhouse.io + renamedValue: "" +resourceTypePrefix: internalvirtualization +rules: + cdi.kubevirt.io: + groupRule: + group: cdi.kubevirt.io + preferredVersion: v1beta1 + renamed: cdi.internal.virtualization.deckhouse.io + versions: + - v1beta1 + resourceRules: + cdiconfigs: + categories: [] + kind: CDIConfig + listKind: CDIConfigList + plural: cdiconfigs + preferredVersion: v1beta1 + shortNames: [] + singular: cdiconfig + versions: + - v1beta1 + cdis: + categories: [] + kind: CDI + listKind: CDIList + plural: cdis + preferredVersion: v1beta1 + shortNames: + - cdi + - cdis + singular: cdi + versions: + - v1beta1 + dataimportcrons: + categories: + - all + kind: DataImportCron + listKind: DataImportCronList + plural: dataimportcrons + preferredVersion: v1beta1 + shortNames: + - dic + - dics + singular: dataimportcron + versions: + - v1beta1 + datasources: + categories: + - all + kind: DataSource + listKind: DataSourceList + plural: datasources + preferredVersion: v1beta1 + shortNames: + - das + singular: datasource + versions: + - v1beta1 + datavolumes: + categories: + - all + kind: DataVolume + listKind: DataVolumeList + plural: datavolumes + preferredVersion: v1beta1 + shortNames: + - dv + - dvs + singular: datavolume + versions: + - v1beta1 + objecttransfers: + categories: [] + kind: ObjectTransfer + listKind: ObjectTransferList + plural: objecttransfers + preferredVersion: v1beta1 + shortNames: + - ot + - ots + singular: objecttransfer + versions: + - v1beta1 + storageprofiles: + categories: [] + kind: StorageProfile + listKind: StorageProfileList + plural: storageprofiles + preferredVersion: v1beta1 + shortNames: [] + singular: storageprofile + versions: + - v1beta1 + volumeclonesources: + categories: [] + kind: VolumeCloneSource + listKind: VolumeCloneSourceList + plural: volumeclonesources + preferredVersion: v1beta1 + shortNames: [] + singular: volumeclonesource + versions: + - v1beta1 + volumeimportsources: + categories: [] + kind: VolumeImportSource + listKind: VolumeImportSourceList + plural: volumeimportsources + preferredVersion: v1beta1 + shortNames: [] + singular: volumeimportsource + versions: + - v1beta1 + volumeuploadsources: + categories: [] + kind: VolumeUploadSource + listKind: VolumeUploadSourceList + plural: volumeuploadsources + preferredVersion: v1beta1 + shortNames: [] + singular: volumeuploadsource + versions: + - v1beta1 + clone.kubevirt.io: + groupRule: + group: clone.kubevirt.io + preferredVersion: v1alpha1 + renamed: clone.internal.virtualization.deckhouse.io + versions: + - v1alpha1 + resourceRules: + virtualmachineclones: + categories: + - all + kind: VirtualMachineClone + listKind: VirtualMachineCloneList + plural: virtualmachineclones + preferredVersion: v1alpha1 + shortNames: + - vmclone + - vmclones + singular: virtualmachineclone + versions: + - v1alpha1 + export.kubevirt.io: + groupRule: + group: export.kubevirt.io + preferredVersion: v1alpha1 + renamed: export.internal.virtualization.deckhouse.io + versions: + - v1alpha1 + resourceRules: + virtualmachineexports: + categories: + - all + kind: VirtualMachineExport + listKind: VirtualMachineExportList + plural: virtualmachineexports + preferredVersion: v1alpha1 + shortNames: + - vmexport + - vmexports + singular: virtualmachineexport + versions: + - v1alpha1 + forklift.cdi.kubevirt.io: + groupRule: + group: forklift.cdi.kubevirt.io + preferredVersion: v1beta1 + renamed: forklift.cdi.internal.virtualization.deckhouse.io + versions: + - v1beta1 + resourceRules: + openstackvolumepopulators: + categories: null + kind: OpenstackVolumePopulator + listKind: OpenstackVolumePopulatorList + plural: openstackvolumepopulators + preferredVersion: v1beta1 + shortNames: + - osvp + - osvps + singular: openstackvolumepopulator + versions: + - v1beta1 + ovirtvolumepopulators: + categories: null + kind: OvirtVolumePopulator + listKind: OvirtVolumePopulatorList + plural: ovirtvolumepopulators + preferredVersion: v1beta1 + shortNames: + - ovvp + - ovvps + singular: ovirtvolumepopulator + versions: + - v1beta1 + instancetype.kubevirt.io: + groupRule: + group: instancetype.kubevirt.io + preferredVersion: v1alpha2 + renamed: instancetype.internal.virtualization.deckhouse.io + versions: + - v1alpha1 + - v1alpha2 + resourceRules: + virtualmachineclusterinstancetypes: + categories: [] + kind: VirtualMachineClusterInstancetype + listKind: VirtualMachineClusterInstancetypeList + plural: virtualmachineclusterinstancetypes + preferredVersion: v1alpha2 + shortNames: + - vmclusterinstancetype + - vmclusterinstancetypes + - vmcf + - vmcfs + singular: virtualmachineclusterinstancetype + versions: + - v1alpha1 + - v1alpha2 + virtualmachineclusterpreferences: + categories: [] + kind: VirtualMachineClusterPreference + listKind: VirtualMachineClusterPreferenceList + plural: virtualmachineclusterpreferences + preferredVersion: v1alpha2 + shortNames: + - vmcp + - vmcps + singular: virtualmachineclusterpreference + versions: + - v1alpha1 + - v1alpha2 + virtualmachineinstancetypes: + categories: + - all + kind: VirtualMachineInstancetype + listKind: VirtualMachineInstancetypeList + plural: virtualmachineinstancetypes + preferredVersion: v1alpha2 + shortNames: + - vminstancetype + - vminstancetypes + - vmf + - vmfs + singular: virtualmachineinstancetype + versions: + - v1alpha1 + - v1alpha2 + virtualmachinepreferences: + categories: + - all + kind: VirtualMachinePreference + listKind: VirtualMachinePreferenceList + plural: virtualmachinepreferences + preferredVersion: v1alpha2 + shortNames: + - vmpref + - vmprefs + - vmp + - vmps + singular: virtualmachinepreference + versions: + - v1alpha1 + - v1alpha2 + kubevirt.io: + groupRule: + group: kubevirt.io + preferredVersion: v1 + renamed: internal.virtualization.deckhouse.io + versions: + - v1 + - v1alpha3 + resourceRules: + kubevirts: + categories: + - all + kind: KubeVirt + listKind: KubeVirtList + plural: kubevirts + preferredVersion: v1 + shortNames: + - kv + - kvs + singular: kubevirt + versions: + - v1 + - v1alpha3 + virtualmachineinstancemigrations: + categories: + - all + kind: VirtualMachineInstanceMigration + listKind: VirtualMachineInstanceMigrationList + plural: virtualmachineinstancemigrations + preferredVersion: v1 + shortNames: + - vmim + - vmims + singular: virtualmachineinstancemigration + versions: + - v1 + - v1alpha3 + virtualmachineinstancepresets: + categories: + - all + kind: VirtualMachineInstancePreset + listKind: VirtualMachineInstancePresetList + plural: virtualmachineinstancepresets + preferredVersion: v1 + shortNames: + - vmipreset + - vmipresets + singular: virtualmachineinstancepreset + versions: + - v1 + - v1alpha3 + virtualmachineinstancereplicasets: + categories: + - all + kind: VirtualMachineInstanceReplicaSet + listKind: VirtualMachineInstanceReplicaSetList + plural: virtualmachineinstancereplicasets + preferredVersion: v1 + shortNames: + - vmirs + - vmirss + singular: virtualmachineinstancereplicaset + versions: + - v1 + - v1alpha3 + virtualmachineinstances: + categories: + - all + kind: VirtualMachineInstance + listKind: VirtualMachineInstanceList + plural: virtualmachineinstances + preferredVersion: v1 + shortNames: + - vmi + - vmsi + singular: virtualmachineinstance + versions: + - v1 + - v1alpha3 + virtualmachines: + categories: + - all + kind: VirtualMachine + listKind: VirtualMachineList + plural: virtualmachines + preferredVersion: v1 + shortNames: + - vm + - vms + singular: virtualmachine + versions: + - v1 + - v1alpha3 + migrations.kubevirt.io: + groupRule: + group: migrations.kubevirt.io + preferredVersion: v1alpha1 + renamed: migrations.internal.virtualization.deckhouse.io + versions: + - v1alpha1 + resourceRules: + migrationpolicies: + categories: + - all + kind: MigrationPolicy + listKind: MigrationPolicyList + plural: migrationpolicies + preferredVersion: v1alpha1 + shortNames: [] + singular: migrationpolicy + versions: + - v1alpha1 + pool.kubevirt.io: + groupRule: + group: pool.kubevirt.io + preferredVersion: v1alpha1 + renamed: pool.internal.virtualization.deckhouse.io + versions: + - v1alpha1 + resourceRules: + virtualmachinepools: + categories: + - all + kind: VirtualMachinePool + listKind: VirtualMachinePoolList + plural: virtualmachinepools + preferredVersion: v1alpha1 + shortNames: + - vmpool + - vmpools + singular: virtualmachinepool + versions: + - v1alpha1 + snapshot.kubevirt.io: + groupRule: + group: snapshot.kubevirt.io + preferredVersion: v1alpha1 + renamed: snapshot.internal.virtualization.deckhouse.io + versions: + - v1alpha1 + resourceRules: + virtualmachinerestores: + categories: + - all + kind: VirtualMachineRestore + listKind: VirtualMachineRestoreList + plural: virtualmachinerestores + preferredVersion: v1alpha1 + shortNames: + - vmrestore + - vmrestores + singular: virtualmachinerestore + versions: + - v1alpha1 + virtualmachinesnapshotcontents: + categories: + - all + kind: VirtualMachineSnapshotContent + listKind: VirtualMachineSnapshotContentList + plural: virtualmachinesnapshotcontents + preferredVersion: v1alpha1 + shortNames: + - vmsnapshotcontent + - vmsnapshotcontents + singular: virtualmachinesnapshotcontent + versions: + - v1alpha1 + virtualmachinesnapshots: + categories: + - all + kind: VirtualMachineSnapshot + listKind: VirtualMachineSnapshotList + plural: virtualmachinesnapshots + preferredVersion: v1alpha1 + shortNames: + - vmsnapshot + - vmsnapshots + singular: virtualmachinesnapshot + versions: + - v1alpha1 +shortNamePrefix: intvirt +webhooks: + /cdi-validate: + group: cdi.kubevirt.io + path: /cdi-validate + resource: cdis + /dataimportcron-validate: + group: cdi.kubevirt.io + path: /dataimportcron-validate + resource: dataimportcrons + /datavolume-mutate: + group: cdi.kubevirt.io + path: /datavolume-mutate + resource: datavolumes + /datavolume-validate: + group: cdi.kubevirt.io + path: /datavolume-validate + resource: datavolumes + /kubevirt-validate-delete: + group: kubevirt.io + path: /kubevirt-validate-delete + resource: kubevirts + /kubevirt-validate-update: + group: kubevirt.io + path: /kubevirt-validate-update + resource: kubevirts + /migration-mutate-create: + group: kubevirt.io + path: /migration-mutate-create + resource: virtualmachineinstancemigrations + /migration-policy-validate-create: + group: migrations.kubevirt.io + path: /migration-policy-validate-create + resource: migrationpolicies + /migration-validate-create: + group: kubevirt.io + path: /migration-validate-create + resource: virtualmachineinstancemigrations + /migration-validate-update: + group: kubevirt.io + path: /migration-validate-update + resource: virtualmachineinstancemigrations + /objecttransfer-validate: + group: cdi.kubevirt.io + path: /objecttransfer-validate + resource: objecttransfers + /populator-validate: + group: cdi.kubevirt.io + path: /populator-validate + resource: volumeimportsources + /status-validate: + group: kubevirt.io + path: /status-validate + resource: virtualmachines/status,virtualmachineinstancereplicasets/status,virtualmachineinstancemigrations/status + /virtualmachineclusterinstancetypes-validate: + group: instancetype.kubevirt.io + path: /virtualmachineclusterinstancetypes-validate + resource: virtualmachineclusterinstancetypes + /virtualmachineclusterpreferences-validate: + group: instancetype.kubevirt.io + path: /virtualmachineclusterpreferences-validate + resource: virtualmachineclusterpreferences + /virtualmachineexports-validate: + group: export.kubevirt.io + path: /virtualmachineexports-validate + resource: virtualmachineexports + /virtualmachineinstances-mutate: + group: kubevirt.io + path: /virtualmachineinstances-mutate + resource: virtualmachineinstances + /virtualmachineinstances-validate-create: + group: kubevirt.io + path: /virtualmachineinstances-validate-create + resource: virtualmachineinstances + /virtualmachineinstances-validate-update: + group: kubevirt.io + path: /virtualmachineinstances-validate-update + resource: virtualmachineinstances + /virtualmachineinstancetypes-validate: + group: instancetype.kubevirt.io + path: /virtualmachineinstancetypes-validate + resource: virtualmachineinstancetypes + /virtualmachinepool-validate: + group: pool.kubevirt.io + path: /virtualmachinepool-validate + resource: virtualmachinepools + /virtualmachinepreferences-validate: + group: instancetype.kubevirt.io + path: /virtualmachinepreferences-validate + resource: virtualmachinepreferences + /virtualmachinereplicaset-validate: + group: kubevirt.io + path: /virtualmachinereplicaset-validate + resource: virtualmachineinstancereplicasets + /virtualmachinerestores-validate: + group: snapshot.kubevirt.io + path: /virtualmachinerestores-validate + resource: virtualmachinerestores + /virtualmachines-mutate: + group: kubevirt.io + path: /virtualmachines-mutate + resource: virtualmachines + /virtualmachines-validate: + group: kubevirt.io + path: /virtualmachines-validate + resource: virtualmachines + /virtualmachinesnapshots-validate: + group: snapshot.kubevirt.io + path: /virtualmachinesnapshots-validate + resource: virtualmachinesnapshots + /vm-clone-mutate-create: + group: clone.kubevirt.io + path: /vm-clone-mutate-create + resource: virtualmachineclones + /vm-clone-validate-create: + group: clone.kubevirt.io + path: /vm-clone-validate-create + resource: virtualmachineclones + /vmipreset-validate: + group: kubevirt.io + path: /vmipreset-validate + resource: virtualmachineinstancepresets +{{- end -}} diff --git a/templates/kubevirt/kubevirt.yaml b/templates/kubevirt/kubevirt.yaml index 47c8ed8b67..ec8610c8ca 100644 --- a/templates/kubevirt/kubevirt.yaml +++ b/templates/kubevirt/kubevirt.yaml @@ -75,22 +75,6 @@ spec: virtualMachineOptions: disableSerialConsoleLog: {} customizeComponents: - flags: - {{- if ne "delve/virt-api" ($delve | dig "debug" "component" "") }} - api: - metrics-listen: 127.0.0.1 - metrics-port: "8080" - {{- end }} - {{- if ne "delve/virt-controller" ($delve | dig "debug" "component" "") }} - controller: - metrics-listen: 127.0.0.1 - metrics-port: "8080" - {{- end }} - {{- if ne "delve/virt-handler" ($delve | dig "debug" "component" "") }} - handler: - metrics-listen: 127.0.0.1 - metrics-port: "8080" - {{- end }} patches: # Add node placement settings for virt-api, virt-controller, virt-operator, virt-handler. - resourceType: Deployment @@ -113,6 +97,10 @@ spec: resourceName: virt-handler patch: '[{"op":"replace","path":"/spec/template/spec/tolerations","value":{{ $tolerationsAnyNode }}}]' type: json + - resourceType: DaemonSet + resourceName: virt-handler + patch: '[{"op":"replace","path":"/spec/template/spec/hostNetwork","value":true}]' + type: json {{- if and $delve (hasKey $delve "debug") }} # Debug {{- if eq $delve.debug.component "delve/virt-api" }} @@ -175,67 +163,25 @@ spec: type: strategic {{- end }} - # Add kube-api-rewriter sidecar containers to virt-controller, virt-api, virt-handler and virt-exportproxy. - - resourceName: virt-controller - resourceType: Deployment - patch: {{ include "kube_api_rewriter.pod_spec_strategic_patch_json" (list . "virt-controller") }} - type: strategic - + # Add kube-api-rewriter sidecar containers to virt-api and virt-exportproxy. {{- $virtApiRewriterSettings := dict }} {{- $_ := set $virtApiRewriterSettings "WEBHOOK_ADDRESS" "https://127.0.0.1:8443" }} {{- $_ := set $virtApiRewriterSettings "WEBHOOK_CERT_FILE" "/etc/virt-api/certificates/tls.crt" }} {{- $_ := set $virtApiRewriterSettings "WEBHOOK_KEY_FILE" "/etc/virt-api/certificates/tls.key" }} {{- $_ := set $virtApiRewriterSettings "webhookCertsVolumeName" "kubevirt-virt-api-certs" }} {{- $_ := set $virtApiRewriterSettings "webhookCertsMountPath" "/etc/virt-api/certificates" }} + {{- $_ := set $virtApiRewriterSettings "healthzPath" "/healthz" }} + {{- $_ := set $virtApiRewriterSettings "readyzPath" "/readyz" }} + {{- $_ := set $virtApiRewriterSettings "healthzPort" 9090 }} + {{- $_ := set $virtApiRewriterSettings "probeScheme" "HTTP" }} + {{- $_ := set $virtApiRewriterSettings "injectPodIP" true }} + {{- $_ := set $virtApiRewriterSettings "monitoringBindAddress" "$(POD_IP):9090" }} + {{- $_ := set $virtApiRewriterSettings "monitoringAuth" (dict "group" "apps" "version" "v1" "resource" "deployments" "namespace" (printf "d8-%s" .Chart.Name) "name" "virt-api" "subresource" "prometheus-metrics") }} - resourceName: virt-api resourceType: Deployment patch: {{ include "kube_api_rewriter.pod_spec_strategic_patch_json" (tuple . "virt-api" $virtApiRewriterSettings) }} type: strategic - - resourceName: virt-handler - resourceType: DaemonSet - patch: {{ include "kube_api_rewriter.pod_spec_strategic_patch_json" (list . "virt-handler") }} - type: strategic - - # Add kube-api-rewriter sidecar containers to virt-controller, virt-api, virt-handler. - {{- $kubeRbacProxySettings := dict }} - {{- $_ := set $kubeRbacProxySettings "runAsUserNobody" true }} - {{- $_ := set $kubeRbacProxySettings "ignorePaths" "/proxy/healthz,/proxy/readyz" }} - {{- $_ := set $kubeRbacProxySettings "upstreams" (list - (dict "upstream" "http://127.0.0.1:9090/metrics" "path" "/proxy/metrics" "name" "kube-api-rewriter") - (dict "upstream" "http://127.0.0.1:8080/metrics" "path" "/metrics" "name" "virt-controller") - (dict "upstream" "http://127.0.0.1:9090/healthz" "path" "/proxy/healthz" "name" "kube-api-rewriter") - (dict "upstream" "http://127.0.0.1:9090/readyz" "path" "/proxy/readyz" "name" "kube-api-rewriter") - ) }} - - resourceName: virt-controller - resourceType: Deployment - patch: {{ include "kube_rbac_proxy.pod_spec_strategic_patch_json" (tuple . $kubeRbacProxySettings) }} - type: strategic - - {{- $_ := set $kubeRbacProxySettings "ignorePaths" "/proxy/healthz,/proxy/readyz" }} - {{- $_ := set $kubeRbacProxySettings "upstreams" (list - (dict "upstream" "http://127.0.0.1:9090/metrics" "path" "/proxy/metrics" "name" "kube-api-rewriter") - (dict "upstream" "http://127.0.0.1:8080/metrics" "path" "/metrics" "name" "virt-api") - (dict "upstream" "http://127.0.0.1:9090/healthz" "path" "/proxy/healthz" "name" "kube-api-rewriter") - (dict "upstream" "http://127.0.0.1:9090/readyz" "path" "/proxy/readyz" "name" "kube-api-rewriter") - ) }} - - resourceName: virt-api - resourceType: Deployment - patch: {{ include "kube_rbac_proxy.pod_spec_strategic_patch_json" (tuple . $kubeRbacProxySettings) }} - type: strategic - - {{- $_ := set $kubeRbacProxySettings "ignorePaths" "/proxy/healthz,/proxy/readyz" }} - {{- $_ := set $kubeRbacProxySettings "upstreams" (list - (dict "upstream" "http://127.0.0.1:9090/metrics" "path" "/proxy/metrics" "resource" "daemonsets" "name" "kube-api-rewriter") - (dict "upstream" "http://127.0.0.1:8080/metrics" "path" "/metrics" "resource" "daemonsets" "name" "virt-handler") - (dict "upstream" "http://127.0.0.1:9090/healthz" "path" "/proxy/healthz" "resource" "daemonsets" "name" "kube-api-rewriter") - (dict "upstream" "http://127.0.0.1:9090/readyz" "path" "/proxy/readyz" "resource" "daemonsets" "name" "kube-api-rewriter") - ) }} - - resourceName: virt-handler - resourceType: DaemonSet - patch: {{ include "kube_rbac_proxy.pod_spec_strategic_patch_json" (tuple . $kubeRbacProxySettings) }} - type: strategic - # Add rewriter proxy container port to Services used by webhook configurations. # First need to set name for existing port to make strategic patch works later. - resourceName: virt-api @@ -330,10 +276,10 @@ spec: resourceName: virt-handler patch: {{ include "pod_spec_priority_class_name_patch" $priorityClassName }} type: strategic - # Patch service for https-metrics + # Patch service to target the main virt-handler port - resourceType: Service resourceName: kubevirt-prometheus-metrics - patch: '[{"op": "replace", "path": "/spec/ports/0/targetPort", "value": "https-metrics"}]' + patch: '[{"op": "replace", "path": "/spec/ports/0/targetPort", "value": "virt-handler"}]' type: json # Additional environment variables for virt-controller. @@ -356,6 +302,32 @@ env: patch: '{"spec":{"template":{"metadata":{"labels":{"security.deckhouse.io/security-policy-exception": "virt-handler-ds"}}}}}' type: strategic + # Expose virt-handler ports: health API (--port) and console server (--console-server-port). + - resourceName: virt-handler + resourceType: DaemonSet + patch: '{"spec":{"template":{"spec":{"containers":[{"name":"virt-handler","ports":[{"containerPort":{{ include "virt_handler.port" . | int }},"name":"virt-handler","protocol":"TCP"},{"containerPort":{{ include "virt_handler.console_server_port" . | int }},"name":"console","protocol":"TCP"}]}]}}}}' + type: strategic + + # Rewrite virt-api args, replacing the default ports baked into the image. + # This is required because customizeComponents.flags only appends flags and cannot replace existing ones. + - resourceName: virt-api + resourceType: Deployment + patch: '{"spec":{"template":{"spec":{"containers":[{"name":"virt-api","args":["--port","8443","--console-server-port","{{ include "virt_handler.console_server_port" . }}","--subresources-only","-v","2"]}]}}}}' + type: strategic + + # Rewrite virt-handler args with hostNetwork ports, replacing the default ports baked into the image. + # This is required because customizeComponents.flags only appends flags and cannot replace existing ones. + - resourceName: virt-handler + resourceType: DaemonSet + patch: '{"spec":{"template":{"spec":{"containers":[{"name":"virt-handler","args":["--port","{{ include "virt_handler.port" . }}","--hostname-override","$(NODE_NAME)","--pod-ip-address","$(MY_POD_IP)","--max-metric-requests","3","--console-server-port","{{ include "virt_handler.console_server_port" . }}","--migration-port-range-enabled","true","--migration-port-range-first","{{ include "virt_handler.migration_port_first" . }}","--migration-port-range-last","{{ include "virt_handler.migration_port_last" . }}","--graceful-shutdown-seconds","315","-v","2"]}]}}}}' + type: strategic + + # Override virt-handler liveness and readiness probes to use the new host-network port. + - resourceName: virt-handler + resourceType: DaemonSet + patch: '{"spec":{"template":{"spec":{"containers":[{"name":"virt-handler","livenessProbe":{"httpGet":{"path":"/healthz","port":{{ include "virt_handler.port" . | int }},"scheme":"HTTPS"},"failureThreshold":3,"initialDelaySeconds":15,"periodSeconds":45,"successThreshold":1,"timeoutSeconds":10},"readinessProbe":{"httpGet":{"path":"/healthz","port":{{ include "virt_handler.port" . | int }},"scheme":"HTTPS"},"failureThreshold":3,"initialDelaySeconds":15,"periodSeconds":20,"successThreshold":1,"timeoutSeconds":10}}]}}}}' + type: strategic + # Change host path for directory with capabilities xml files. We have custom qemu with different # machine types thus it conflicts with the original kubevirt. - resourceName: virt-handler @@ -363,6 +335,16 @@ env: patch: '{"spec":{"template":{"spec":{"volumes":[{"name":"node-labeller","hostPath":{"path":"/var/run/d8-virtualization/node-labeller"}}]}}}}' type: strategic + # Mount kube-api-rewriter rules ConfigMap into virt-controller and virt-handler. + - resourceName: virt-controller + resourceType: Deployment + patch: '{"spec":{"template":{"metadata":{"annotations":{"checksum/rewrite-rules":"{{ include "kubevirt.rewrite_rules_data" . | sha256sum }}"}},"spec":{"volumes":[{"name":"kube-api-rewriter-rules","configMap":{"name":"kube-api-rewriter-rules"}}],"containers":[{"name":"virt-controller","volumeMounts":[{"name":"kube-api-rewriter-rules","mountPath":"/etc/kube-api-rewriter","readOnly":true}]}]}}}}' + type: strategic + - resourceName: virt-handler + resourceType: DaemonSet + patch: '{"spec":{"template":{"metadata":{"annotations":{"checksum/rewrite-rules":"{{ include "kubevirt.rewrite_rules_data" . | sha256sum }}"}},"spec":{"volumes":[{"name":"kube-api-rewriter-rules","configMap":{"name":"kube-api-rewriter-rules"}}],"containers":[{"name":"virt-handler","volumeMounts":[{"name":"kube-api-rewriter-rules","mountPath":"/etc/kube-api-rewriter","readOnly":true}]}]}}}}' + type: strategic + imagePullPolicy: IfNotPresent imagePullSecrets: - name: virtualization-module-registry diff --git a/templates/kubevirt/virt-handler/rewrite-rules-cm.yaml b/templates/kubevirt/virt-handler/rewrite-rules-cm.yaml new file mode 100644 index 0000000000..0d0c5bf6e1 --- /dev/null +++ b/templates/kubevirt/virt-handler/rewrite-rules-cm.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: kube-api-rewriter-rules + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list .) | nindent 2 }} +data: + rewrite-rules.yaml: | +{{ include "kubevirt.rewrite_rules_data" . | indent 4 }} diff --git a/templates/kubevirt/virt-operator/deployment.yaml b/templates/kubevirt/virt-operator/deployment.yaml index 833ef6ccf3..2afa4e3c7d 100644 --- a/templates/kubevirt/virt-operator/deployment.yaml +++ b/templates/kubevirt/virt-operator/deployment.yaml @@ -33,7 +33,6 @@ spec: resourcePolicy: containerPolicies: {{- include "kube_api_rewriter.vpa_container_policy" . | nindent 4 }} - {{- include "kube_rbac_proxy.vpa_container_policy" . | nindent 4 }} - containerName: virt-operator minAllowed: {{- include "virt_operator_resources" . | nindent 8 }} @@ -95,26 +94,19 @@ spec: {{- $_ := set $rewriterSettings "WEBHOOK_KEY_FILE" "/etc/virt-operator/certificates/tls.key" }} {{- $_ := set $rewriterSettings "webhookCertsVolumeName" "kubevirt-operator-certs" }} {{- $_ := set $rewriterSettings "webhookCertsMountPath" "/etc/virt-operator/certificates" }} + {{- $_ := set $rewriterSettings "healthzPath" "/healthz" }} + {{- $_ := set $rewriterSettings "readyzPath" "/readyz" }} + {{- $_ := set $rewriterSettings "healthzPort" 9090 }} + {{- $_ := set $rewriterSettings "probeScheme" "HTTP" }} + {{- $_ := set $rewriterSettings "injectPodIP" true }} + {{- $_ := set $rewriterSettings "monitoringBindAddress" "$(POD_IP):9090" }} + {{- $_ := set $rewriterSettings "monitoringAuth" (dict "group" "apps" "version" "v1" "resource" "deployments" "namespace" (printf "d8-%s" .Chart.Name) "name" "virt-operator" "subresource" "prometheus-metrics") }} {{- include "kube_api_rewriter.sidecar_container" (tuple . $rewriterSettings) | nindent 6 }} - {{- $kubeRbacProxySettings := dict }} - {{- $_ := set $kubeRbacProxySettings "runAsUserNobody" true }} - {{- $_ := set $kubeRbacProxySettings "ignorePaths" "/proxy/healthz,/proxy/readyz" }} - {{- $_ := set $kubeRbacProxySettings "upstreams" (list - (dict "upstream" "http://127.0.0.1:9090/metrics" "path" "/proxy/metrics" "name" "kube-api-rewriter") - (dict "upstream" "http://127.0.0.1:8080/metrics" "path" "/metrics" "name" "virt-operator") - (dict "upstream" "http://127.0.0.1:9090/healthz" "path" "/proxy/healthz" "name" "kube-api-rewriter") - (dict "upstream" "http://127.0.0.1:9090/readyz" "path" "/proxy/readyz" "name" "kube-api-rewriter") - ) }} - {{- include "kube_rbac_proxy.sidecar_container" (tuple . $kubeRbacProxySettings) | nindent 6 }} - name: virt-operator {{- include "helm_lib_module_container_security_context_read_only_root_filesystem_capabilities_drop_all_pss_restricted" . | nindent 8 }} args: - --port - "8443" - - --metrics-listen - - 127.0.0.1 - - --metrics-port - - "8080" - -v - "2" command: @@ -136,13 +128,13 @@ spec: httpGet: path: /healthz port: 8443 - scheme: HTTP + scheme: HTTPS initialDelaySeconds: 10 livenessProbe: httpGet: path: /healthz port: 8443 - scheme: HTTP + scheme: HTTPS initialDelaySeconds: 10 resources: requests: diff --git a/templates/virtualization-dra/daemonset.yaml b/templates/virtualization-dra/daemonset.yaml index 16b04b81f0..871c2845f8 100644 --- a/templates/virtualization-dra/daemonset.yaml +++ b/templates/virtualization-dra/daemonset.yaml @@ -167,7 +167,8 @@ spec: args: - {{ include "virtualization-dra.featureGates" . }} {{/* https://github.com/deckhouse/deckhouse/pull/18139 */}} - - --usbipd-port=4280 + - --usbipd-port={{ include "virtualization_dra.usbipd_port" . }} + - --healthz-port={{ include "virtualization_dra.health_port" . }} {{- if eq (include "moduleLogLevel" .) "debug" }} - --log-level=debug - --log-debug-verbosity=10 @@ -189,23 +190,23 @@ spec: {{- include "virtualization-dra_resources" . | nindent 14 }} {{- end }} ports: - - containerPort: 4280 + - containerPort: {{ include "virtualization_dra.usbipd_port" . }} name: usbipd protocol: TCP - - containerPort: 51515 + - containerPort: {{ include "virtualization_dra.health_port" . }} name: health protocol: TCP {{- include "delvePorts" (list $delve "delve/virtualization-dra") | nindent 12 }} {{- if ne "delve/virtualization-dra" ($delve | dig "debug" "component" "") }} readinessProbe: grpc: - port: 51515 + port: {{ include "virtualization_dra.health_port" . }} service: liveness failureThreshold: 3 periodSeconds: 10 livenessProbe: grpc: - port: 51515 + port: {{ include "virtualization_dra.health_port" . }} service: liveness failureThreshold: 3 periodSeconds: 10 diff --git a/templates/vm-route-forge/daemonset.yaml b/templates/vm-route-forge/daemonset.yaml index e0816552e8..62b08cbc13 100644 --- a/templates/vm-route-forge/daemonset.yaml +++ b/templates/vm-route-forge/daemonset.yaml @@ -122,10 +122,10 @@ spec: {{- end }} {{- if eq (include "moduleLogLevel" .) "debug" }} - name: PPROF_BIND_ADDRESS - value: ":8119" + value: ":{{ include "vm_route_forge.pprof_port" . }}" {{- end }} - name: HEALTH_PROBE_BIND_ADDRESS - value: "127.0.0.1:8118" + value: "127.0.0.1:{{ include "vm_route_forge.health_port" . }}" resources: requests: {{- include "helm_lib_module_ephemeral_storage_only_logs" . | nindent 14 }} @@ -133,10 +133,10 @@ spec: {{- include "vm-route-forge_resources" . | nindent 14 }} {{- end }} ports: - - containerPort: 8119 + - containerPort: {{ include "vm_route_forge.pprof_port" . }} name: pprof protocol: TCP - - containerPort: 8118 + - containerPort: {{ include "vm_route_forge.health_port" . }} name: health protocol: TCP {{- include "delvePorts" (list $delve "delve/vm-route-forge") | nindent 12 }} @@ -145,7 +145,7 @@ spec: httpGet: host: localhost path: /readyz - port: 8118 + port: {{ include "vm_route_forge.health_port" . }} scheme: HTTP initialDelaySeconds: 5 failureThreshold: 2 @@ -154,7 +154,7 @@ spec: httpGet: host: localhost path: /healthz - port: 8118 + port: {{ include "vm_route_forge.health_port" . }} scheme: HTTP periodSeconds: 1 failureThreshold: 3 diff --git a/templates/vm-route-forge/service.yaml b/templates/vm-route-forge/service.yaml index e04086d5a0..df5a67f28b 100644 --- a/templates/vm-route-forge/service.yaml +++ b/templates/vm-route-forge/service.yaml @@ -9,7 +9,7 @@ metadata: spec: ports: - name: pprof - port: 8119 + port: {{ include "vm_route_forge.pprof_port" . }} protocol: TCP targetPort: pprof selector: