From 052ba12409d653a8b5268781c11365ed5cbb5c83 Mon Sep 17 00:00:00 2001 From: matthalp Date: Fri, 6 Feb 2026 17:10:53 -0800 Subject: [PATCH 01/15] cmd/k8s-operator: add accept-app-caps annotation for Ingress resources Add support for the tailscale.com/accept-app-caps annotation on Ingress resources. This populates the AcceptAppCaps field on HTTPHandler entries in the serve config, which causes the serve proxy to forward matching peer capabilities in the Tailscale-App-Capabilities header to backends. The annotation accepts a comma-separated list of capability names (e.g. "example.com/cap/monitoring,example.com/cap/admin"). Each capability is validated against the standard app capability regex. Invalid capabilities are skipped with a warning event, consistent with the operator's soft-validation pattern. Both the standard Ingress reconciler and the HA (ProxyGroup) Ingress reconciler benefit from this change since they share the same handlersForIngress() function. Updates #tailscale/corp#28049 Signed-off-by: matthalp Co-Authored-By: Claude Opus 4.6 --- cmd/k8s-operator/ingress-for-pg.go | 11 ++ cmd/k8s-operator/ingress-for-pg_test.go | 95 ++++++++++++++++ cmd/k8s-operator/ingress.go | 35 +++++- cmd/k8s-operator/ingress_test.go | 139 ++++++++++++++++++++++++ cmd/k8s-operator/sts.go | 5 +- 5 files changed, 282 insertions(+), 3 deletions(-) diff --git a/cmd/k8s-operator/ingress-for-pg.go b/cmd/k8s-operator/ingress-for-pg.go index 28a836e975273..072995a956f05 100644 --- a/cmd/k8s-operator/ingress-for-pg.go +++ b/cmd/k8s-operator/ingress-for-pg.go @@ -646,6 +646,7 @@ func (r *HAIngressReconciler) shouldExpose(ing *networkingv1.Ingress) bool { // validateIngress validates that the Ingress is properly configured. // Currently validates: // - Any tags provided via tailscale.com/tags annotation are valid Tailscale ACL tags +// - Any accept-app-caps provided via tailscale.com/accept-app-caps annotation are valid capability names // - The derived hostname is a valid DNS label // - The referenced ProxyGroup exists and is of type 'ingress' // - Ingress' TLS block is invalid @@ -658,6 +659,16 @@ func (r *HAIngressReconciler) validateIngress(ctx context.Context, ing *networki errs = append(errs, fmt.Errorf("Ingress contains invalid tags: %v", strings.Join(violations, ","))) } + // Validate accept-app-caps if present + if raw, ok := ing.Annotations[AnnotationAcceptAppCaps]; ok && raw != "" { + for _, p := range strings.Split(raw, ",") { + p = strings.TrimSpace(p) + if p != "" && !validAppCap.MatchString(p) { + errs = append(errs, fmt.Errorf("invalid app capability %q", p)) + } + } + } + // Validate TLS configuration if len(ing.Spec.TLS) > 0 && (len(ing.Spec.TLS) > 1 || len(ing.Spec.TLS[0].Hosts) > 1) { errs = append(errs, fmt.Errorf("Ingress contains invalid TLS block %v: only a single TLS entry with a single host is allowed", ing.Spec.TLS)) diff --git a/cmd/k8s-operator/ingress-for-pg_test.go b/cmd/k8s-operator/ingress-for-pg_test.go index 33e27ef371d90..37bd25d5dbc45 100644 --- a/cmd/k8s-operator/ingress-for-pg_test.go +++ b/cmd/k8s-operator/ingress-for-pg_test.go @@ -1187,6 +1187,101 @@ func createPGResources(t *testing.T, fc client.Client, pgName string) { } } +func TestIngressPGReconciler_AcceptAppCaps(t *testing.T) { + ingPGR, fc, ft := setupIngressTest(t) + + // Create backend Service that the Ingress will route to + backendSvc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + }, + Spec: corev1.ServiceSpec{ + ClusterIP: "10.0.0.1", + Ports: []corev1.ServicePort{ + { + Port: 8080, + }, + }, + }, + } + mustCreate(t, fc, backendSvc) + + // Create test Ingress with accept-app-caps annotation + ing := &networkingv1.Ingress{ + TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ingress", + Namespace: "default", + UID: types.UID("1234-UID"), + Annotations: map[string]string{ + "tailscale.com/proxy-group": "test-pg", + "tailscale.com/accept-app-caps": "example.com/cap/monitoring,example.com/cap/admin", + }, + }, + Spec: networkingv1.IngressSpec{ + IngressClassName: ptr.To("tailscale"), + DefaultBackend: &networkingv1.IngressBackend{ + Service: &networkingv1.IngressServiceBackend{ + Name: "test", + Port: networkingv1.ServiceBackendPort{ + Number: 8080, + }, + }, + }, + TLS: []networkingv1.IngressTLS{ + {Hosts: []string{"my-svc"}}, + }, + }, + } + if err := fc.Create(context.Background(), ing); err != nil { + t.Fatal(err) + } + + // Reconcile + expectReconciled(t, ingPGR, "default", "test-ingress") + populateTLSSecret(t, fc, "test-pg", "my-svc.ts.net") + expectReconciled(t, ingPGR, "default", "test-ingress") + + // Verify Tailscale Service + verifyTailscaleService(t, ft, "svc:my-svc", []string{"tcp:443"}) + + // Verify the serve config has AcceptAppCaps on handlers + cm := &corev1.ConfigMap{} + if err := fc.Get(context.Background(), types.NamespacedName{ + Name: "test-pg-ingress-config", + Namespace: "operator-ns", + }, cm); err != nil { + t.Fatalf("getting ConfigMap: %v", err) + } + + cfg := &ipn.ServeConfig{} + if err := json.Unmarshal(cm.BinaryData["serve-config.json"], cfg); err != nil { + t.Fatalf("unmarshaling serve config: %v", err) + } + + svc := cfg.Services[tailcfg.ServiceName("svc:my-svc")] + if svc == nil { + t.Fatal("service svc:my-svc not found in serve config") + } + + ep := ipn.HostPort("my-svc.ts.net:443") + webCfg := svc.Web[ep] + if webCfg == nil { + t.Fatalf("web config for %q not found", ep) + } + + handler := webCfg.Handlers["/"] + if handler == nil { + t.Fatal("handler for path / not found") + } + + wantCaps := []tailcfg.PeerCapability{"example.com/cap/monitoring", "example.com/cap/admin"} + if !reflect.DeepEqual(handler.AcceptAppCaps, wantCaps) { + t.Errorf("AcceptAppCaps = %v, want %v", handler.AcceptAppCaps, wantCaps) + } +} + func setupIngressTest(t *testing.T) (*HAIngressReconciler, client.Client, *fakeTSClient) { tsIngressClass := &networkingv1.IngressClass{ ObjectMeta: metav1.ObjectMeta{Name: "tailscale"}, diff --git a/cmd/k8s-operator/ingress.go b/cmd/k8s-operator/ingress.go index 4952e789f6a02..a3078dcb48f3a 100644 --- a/cmd/k8s-operator/ingress.go +++ b/cmd/k8s-operator/ingress.go @@ -8,6 +8,7 @@ package main import ( "context" "fmt" + "regexp" "slices" "strings" "sync" @@ -25,6 +26,7 @@ import ( "tailscale.com/ipn" "tailscale.com/kube/kubetypes" + "tailscale.com/tailcfg" "tailscale.com/types/opt" "tailscale.com/util/clientmetric" "tailscale.com/util/mak" @@ -320,7 +322,37 @@ func validateIngressClass(ctx context.Context, cl client.Client, ingressClassNam return nil } +// validAppCap matches application capability names of the form {domain}/{name}. +// Both parts must use the (simplified) FQDN label character set. +// The "name" can contain forward slashes. +var validAppCap = regexp.MustCompile(`^([\pL\pN-]+\.)+[\pL\pN-]+\/[\pL\pN-/]+$`) + +// parseAcceptAppCaps reads the AnnotationAcceptAppCaps annotation from the +// Ingress, splits it by comma, validates each capability name, and returns the +// valid ones. Invalid capabilities are skipped with a warning event. +func parseAcceptAppCaps(ing *networkingv1.Ingress, rec record.EventRecorder) []tailcfg.PeerCapability { + raw, ok := ing.Annotations[AnnotationAcceptAppCaps] + if !ok || raw == "" { + return nil + } + parts := strings.Split(raw, ",") + var caps []tailcfg.PeerCapability + for _, p := range parts { + p = strings.TrimSpace(p) + if p == "" { + continue + } + if !validAppCap.MatchString(p) { + rec.Eventf(ing, corev1.EventTypeWarning, "InvalidAppCapability", "ignoring invalid app capability %q", p) + continue + } + caps = append(caps, tailcfg.PeerCapability(p)) + } + return caps +} + func handlersForIngress(ctx context.Context, ing *networkingv1.Ingress, cl client.Client, rec record.EventRecorder, tlsHost string, logger *zap.SugaredLogger) (handlers map[string]*ipn.HTTPHandler, err error) { + acceptAppCaps := parseAcceptAppCaps(ing, rec) addIngressBackend := func(b *networkingv1.IngressBackend, path string) { if path == "" { path = "/" @@ -364,7 +396,8 @@ func handlersForIngress(ctx context.Context, ing *networkingv1.Ingress, cl clien proto = "https+insecure://" } mak.Set(&handlers, path, &ipn.HTTPHandler{ - Proxy: proto + svc.Spec.ClusterIP + ":" + fmt.Sprint(port) + path, + Proxy: proto + svc.Spec.ClusterIP + ":" + fmt.Sprint(port) + path, + AcceptAppCaps: acceptAppCaps, }) } addIngressBackend(ing.Spec.DefaultBackend, "/") diff --git a/cmd/k8s-operator/ingress_test.go b/cmd/k8s-operator/ingress_test.go index 1381193065093..0609cdbef9ea5 100644 --- a/cmd/k8s-operator/ingress_test.go +++ b/cmd/k8s-operator/ingress_test.go @@ -24,6 +24,7 @@ import ( "tailscale.com/ipn" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/kubetypes" + "tailscale.com/tailcfg" "tailscale.com/tstest" "tailscale.com/util/mak" ) @@ -937,3 +938,141 @@ func TestTailscaleIngressWithHTTPRedirect(t *testing.T) { t.Errorf("incorrect status ports after removing redirect: got %v, want %v", ing.Status.LoadBalancer.Ingress[0].Ports, wantPorts) } } + +func TestTailscaleIngressWithAcceptAppCaps(t *testing.T) { + fc := fake.NewFakeClient(ingressClass()) + ft := &fakeTSClient{} + fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}} + zl, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } + ingR := &IngressReconciler{ + Client: fc, + ingressClassName: "tailscale", + ssr: &tailscaleSTSReconciler{ + Client: fc, + tsClient: ft, + tsnetServer: fakeTsnetServer, + defaultTags: []string{"tag:k8s"}, + operatorNamespace: "operator-ns", + proxyImage: "tailscale/tailscale", + }, + logger: zl.Sugar(), + } + + // 1. Create Ingress with accept-app-caps annotation + ing := ingress() + mak.Set(&ing.Annotations, AnnotationAcceptAppCaps, "example.com/cap/monitoring,example.com/cap/admin") + mustCreate(t, fc, ing) + mustCreate(t, fc, service()) + + expectReconciled(t, ingR, "default", "test") + + fullName, shortName := findGenName(t, fc, "default", "test", "ingress") + wantCaps := []tailcfg.PeerCapability{"example.com/cap/monitoring", "example.com/cap/admin"} + opts := configOpts{ + replicas: ptr.To[int32](1), + stsName: shortName, + secretName: fullName, + namespace: "default", + parentType: "ingress", + hostname: "default-test", + app: kubetypes.AppIngressResource, + serveConfig: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{ + 443: {HTTPS: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "${TS_CERT_DOMAIN}:443": {Handlers: map[string]*ipn.HTTPHandler{ + "/": { + Proxy: "http://1.2.3.4:8080/", + AcceptAppCaps: wantCaps, + }, + }}, + }, + }, + } + + expectEqual(t, fc, expectedSecret(t, fc, opts)) + expectEqual(t, fc, expectedHeadlessService(shortName, "ingress")) + expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeResourceReqs) +} + +func TestParseAcceptAppCaps(t *testing.T) { + tests := []struct { + name string + annotation string + wantCaps []tailcfg.PeerCapability + wantEvents int // number of warning events expected + }{ + { + name: "empty", + annotation: "", + wantCaps: nil, + }, + { + name: "single_valid", + annotation: "example.com/cap/monitoring", + wantCaps: []tailcfg.PeerCapability{"example.com/cap/monitoring"}, + }, + { + name: "multiple_valid", + annotation: "example.com/cap/monitoring,example.com/cap/admin", + wantCaps: []tailcfg.PeerCapability{ + "example.com/cap/monitoring", + "example.com/cap/admin", + }, + }, + { + name: "whitespace", + annotation: " example.com/cap/monitoring , example.com/cap/admin ", + wantCaps: []tailcfg.PeerCapability{ + "example.com/cap/monitoring", + "example.com/cap/admin", + }, + }, + { + name: "invalid_skipped", + annotation: "example.com/cap/valid,not-a-cap,another.com/cap/ok", + wantCaps: []tailcfg.PeerCapability{ + "example.com/cap/valid", + "another.com/cap/ok", + }, + wantEvents: 1, + }, + { + name: "all_invalid", + annotation: "bad,also-bad", + wantCaps: nil, + wantEvents: 2, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rec := record.NewFakeRecorder(10) + ing := &networkingv1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + }, + } + if tt.annotation != "" { + mak.Set(&ing.Annotations, AnnotationAcceptAppCaps, tt.annotation) + } + got := parseAcceptAppCaps(ing, rec) + if !reflect.DeepEqual(got, tt.wantCaps) { + t.Errorf("parseAcceptAppCaps() = %v, want %v", got, tt.wantCaps) + } + // Drain events and count warnings + close(rec.Events) + var gotEvents int + for range rec.Events { + gotEvents++ + } + if gotEvents != tt.wantEvents { + t.Errorf("got %d warning events, want %d", gotEvents, tt.wantEvents) + } + }) + } +} diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index 519f81fe0db29..f38e43f1ea613 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -69,8 +69,9 @@ const ( AnnotationProxyGroup = "tailscale.com/proxy-group" // Annotations settable by users on ingresses. - AnnotationFunnel = "tailscale.com/funnel" - AnnotationHTTPRedirect = "tailscale.com/http-redirect" + AnnotationFunnel = "tailscale.com/funnel" + AnnotationHTTPRedirect = "tailscale.com/http-redirect" + AnnotationAcceptAppCaps = "tailscale.com/accept-app-caps" // If set to true, set up iptables/nftables rules in the proxy forward // cluster traffic to the tailnet IP of that proxy. This can only be set From 5a2b068aa9851a53b72268aaefe082ff360925f9 Mon Sep 17 00:00:00 2001 From: Ryan Mulligan Date: Wed, 25 Mar 2026 12:53:20 -0700 Subject: [PATCH 02/15] cmd/k8s-operator: support custom TLS Secrets for Ingress --- cmd/k8s-operator/api-server-proxy-pg.go | 2 +- cmd/k8s-operator/api-server-proxy-pg_test.go | 4 +- cmd/k8s-operator/ingress-for-pg.go | 64 +++-- cmd/k8s-operator/ingress-for-pg_test.go | 80 ++++++- cmd/k8s-operator/ingress.go | 56 +++-- cmd/k8s-operator/ingress_test.go | 161 ++++++++++++- cmd/k8s-operator/ingress_tls.go | 239 +++++++++++++++++++ cmd/k8s-operator/operator.go | 5 + cmd/k8s-operator/proxygroup.go | 7 +- cmd/k8s-operator/proxygroup_specs.go | 15 +- cmd/k8s-operator/proxygroup_test.go | 17 +- cmd/k8s-operator/sts.go | 20 +- cmd/k8s-operator/testutils_test.go | 39 +-- ipn/ipnlocal/serve.go | 7 +- ipn/ipnlocal/serve_test.go | 58 +++++ 15 files changed, 704 insertions(+), 70 deletions(-) create mode 100644 cmd/k8s-operator/ingress_tls.go diff --git a/cmd/k8s-operator/api-server-proxy-pg.go b/cmd/k8s-operator/api-server-proxy-pg.go index 0900fd0aaa264..db3299bd32333 100644 --- a/cmd/k8s-operator/api-server-proxy-pg.go +++ b/cmd/k8s-operator/api-server-proxy-pg.go @@ -331,7 +331,7 @@ func (r *KubeAPIServerTSServiceReconciler) deleteFinalizer(ctx context.Context, } func (r *KubeAPIServerTSServiceReconciler) ensureCertResources(ctx context.Context, pg *tsapi.ProxyGroup, domain string) error { - secret := certSecret(pg.Name, r.tsNamespace, domain, pg) + secret := certSecret(pg.Name, r.tsNamespace, domain, pg, nil) if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, secret, func(s *corev1.Secret) { s.Labels = secret.Labels }); err != nil { diff --git a/cmd/k8s-operator/api-server-proxy-pg_test.go b/cmd/k8s-operator/api-server-proxy-pg_test.go index 52dda93e515ee..43df670ad89d7 100644 --- a/cmd/k8s-operator/api-server-proxy-pg_test.go +++ b/cmd/k8s-operator/api-server-proxy-pg_test.go @@ -181,7 +181,7 @@ func TestAPIServerProxyReconciler(t *testing.T) { expectedCfg.APIServerProxy.ServiceName = new(tailcfg.ServiceName("svc:" + pgName)) expectCfg(&expectedCfg) - expectEqual(t, fc, certSecret(pgName, ns, defaultDomain, pg)) + expectEqual(t, fc, certSecret(pgName, ns, defaultDomain, pg, nil)) expectEqual(t, fc, certSecretRole(pgName, ns, defaultDomain)) expectEqual(t, fc, certSecretRoleBinding(pg, ns, defaultDomain)) @@ -243,7 +243,7 @@ func TestAPIServerProxyReconciler(t *testing.T) { pg.Status.URL = "" expectEqual(t, fc, pg, omitPGStatusConditionMessages) - expectEqual(t, fc, certSecret(pgName, ns, updatedDomain, pg)) + expectEqual(t, fc, certSecret(pgName, ns, updatedDomain, pg, nil)) expectEqual(t, fc, certSecretRole(pgName, ns, updatedDomain)) expectEqual(t, fc, certSecretRoleBinding(pg, ns, updatedDomain)) expectMissing[corev1.Secret](t, fc, ns, defaultDomain) diff --git a/cmd/k8s-operator/ingress-for-pg.go b/cmd/k8s-operator/ingress-for-pg.go index 072995a956f05..acdd29b111594 100644 --- a/cmd/k8s-operator/ingress-for-pg.go +++ b/cmd/k8s-operator/ingress-for-pg.go @@ -192,8 +192,12 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin r.recorder.Event(ing, corev1.EventTypeWarning, "InvalidIngressConfiguration", err.Error()) return false, nil } + customTLS, err := customTLSForIngress(ctx, r.Client, ing) + if err != nil { + return false, fmt.Errorf("failed to configure custom TLS for Ingress: %w", err) + } - if !IsHTTPSEnabledOnTailnet(r.tsnetServer) { + if customTLS == nil && !IsHTTPSEnabledOnTailnet(r.tsnetServer) { r.recorder.Event(ing, corev1.EventTypeWarning, "HTTPSNotEnabled", "HTTPS is not enabled on the tailnet; ingress may not work") } @@ -250,8 +254,12 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin if err != nil { return false, fmt.Errorf("error determining DNS name for service: %w", err) } + httpsHost := dnsName + if customTLS != nil { + httpsHost = customTLS.host + } - if err = r.ensureCertResources(ctx, pg, dnsName, ing); err != nil { + if err = r.ensureCertResources(ctx, pg, httpsHost, ing, customTLS); err != nil { return false, fmt.Errorf("error ensuring cert resources: %w", err) } @@ -264,8 +272,8 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin logger.Infof("no Ingress serve config ConfigMap found, unable to update serve config. Ensure that ProxyGroup is healthy.") return svcsChanged, nil } - ep := ipn.HostPort(fmt.Sprintf("%s:443", dnsName)) - handlers, err := handlersForIngress(ctx, ing, r.Client, r.recorder, dnsName, logger) + ep := ipn.HostPort(fmt.Sprintf("%s:443", httpsHost)) + handlers, err := handlersForIngress(ctx, ing, r.Client, r.recorder, httpsHost, logger) if err != nil { return false, fmt.Errorf("failed to get handlers for Ingress: %w", err) } @@ -285,7 +293,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin // Add HTTP endpoint if configured. if isHTTPEndpointEnabled(ing) { logger.Infof("exposing Ingress over HTTP") - epHTTP := ipn.HostPort(fmt.Sprintf("%s:80", dnsName)) + epHTTP := ipn.HostPort(fmt.Sprintf("%s:80", httpsHost)) ingCfg.TCP[80] = &ipn.TCPPortHandler{ HTTP: true, } @@ -297,7 +305,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin } } else if isHTTPRedirectEnabled(ing) { logger.Infof("HTTP redirect enabled, setting up port 80 redirect handlers") - epHTTP := ipn.HostPort(fmt.Sprintf("%s:80", dnsName)) + epHTTP := ipn.HostPort(fmt.Sprintf("%s:80", httpsHost)) ingCfg.TCP[80] = &ipn.TCPPortHandler{HTTP: true} ingCfg.Web[epHTTP] = &ipn.WebServerConfig{ Handlers: map[string]*ipn.HTTPHandler{}, @@ -370,7 +378,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin if isHTTPEndpointEnabled(ing) || isHTTPRedirectEnabled(ing) { mode = serviceAdvertisementHTTPAndHTTPS } - if err = r.maybeUpdateAdvertiseServicesConfig(ctx, serviceName, mode, pg); err != nil { + if err = r.maybeUpdateAdvertiseServicesConfig(ctx, serviceName, mode, pg, httpsHost); err != nil { return false, fmt.Errorf("failed to update tailscaled config: %w", err) } @@ -387,7 +395,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin ing.Status.LoadBalancer.Ingress = nil default: var ports []networkingv1.IngressPortStatus - hasCerts, err := hasCerts(ctx, r.Client, r.tsNamespace, serviceName, pg) + hasCerts, err := hasTLSSecretData(ctx, r.Client, r.tsNamespace, httpsHost) if err != nil { return false, fmt.Errorf("error checking TLS credentials provisioned for Ingress: %w", err) } @@ -407,7 +415,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin // Set Ingress status hostname only if either port 443 or 80 is advertised. var hostname string if len(ports) != 0 { - hostname = dnsName + hostname = httpsHost } ing.Status.LoadBalancer.Ingress = []networkingv1.IngressLoadBalancerIngress{ { @@ -485,7 +493,7 @@ func (r *HAIngressReconciler) maybeCleanupProxyGroup(ctx context.Context, logger } // Make sure the Tailscale Service is not advertised in tailscaled or serve config. - if err = r.maybeUpdateAdvertiseServicesConfig(ctx, tsSvcName, serviceAdvertisementOff, pg); err != nil { + if err = r.maybeUpdateAdvertiseServicesConfig(ctx, tsSvcName, serviceAdvertisementOff, pg, ""); err != nil { return false, fmt.Errorf("failed to update tailscaled config services: %w", err) } @@ -571,7 +579,7 @@ func (r *HAIngressReconciler) maybeCleanup(ctx context.Context, hostname string, } // 4. Unadvertise the Tailscale Service in tailscaled config. - if err = r.maybeUpdateAdvertiseServicesConfig(ctx, serviceName, serviceAdvertisementOff, pg); err != nil { + if err = r.maybeUpdateAdvertiseServicesConfig(ctx, serviceName, serviceAdvertisementOff, pg, ""); err != nil { return false, fmt.Errorf("failed to update tailscaled config services: %w", err) } @@ -766,7 +774,7 @@ const ( serviceAdvertisementHTTPAndHTTPS // Both ports 80 and 443 should be advertised ) -func (r *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Context, serviceName tailcfg.ServiceName, mode serviceAdvertisementMode, pg *tsapi.ProxyGroup) (err error) { +func (r *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Context, serviceName tailcfg.ServiceName, mode serviceAdvertisementMode, pg *tsapi.ProxyGroup, httpsHost string) (err error) { // Get all config Secrets for this ProxyGroup. secrets := &corev1.SecretList{} if err := r.List(ctx, secrets, client.InNamespace(r.tsNamespace), client.MatchingLabels(pgSecretLabels(pg.Name, kubetypes.LabelSecretTypeConfig))); err != nil { @@ -781,7 +789,13 @@ func (r *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con // The only exception is Ingresses with an HTTP endpoint enabled - if an // Ingress has an HTTP endpoint enabled, it will be advertised even if the // TLS cert is not yet provisioned. - hasCert, err := hasCerts(ctx, r.Client, r.tsNamespace, serviceName, pg) + if httpsHost == "" { + httpsHost, err = dnsNameForService(ctx, r.Client, serviceName, pg, r.tsNamespace) + if err != nil { + return fmt.Errorf("error determining TLS hostname for service %q: %w", serviceName, err) + } + } + hasCert, err := hasTLSSecretData(ctx, r.Client, r.tsNamespace, httpsHost) if err != nil { return fmt.Errorf("error checking TLS credentials provisioned for service %q: %w", serviceName, err) } @@ -948,12 +962,16 @@ func ownersAreSetAndEqual(a, b *tailscale.VIPService) bool { // (domain) is a valid Kubernetes resource name. // https://github.com/tailscale/tailscale/blob/8b1e7f646ee4730ad06c9b70c13e7861b964949b/util/dnsname/dnsname.go#L99 // https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-subdomain-names -func (r *HAIngressReconciler) ensureCertResources(ctx context.Context, pg *tsapi.ProxyGroup, domain string, ing *networkingv1.Ingress) error { - secret := certSecret(pg.Name, r.tsNamespace, domain, ing) +func (r *HAIngressReconciler) ensureCertResources(ctx context.Context, pg *tsapi.ProxyGroup, domain string, ing *networkingv1.Ingress, customTLS *ingressCustomTLS) error { + secret := certSecret(pg.Name, r.tsNamespace, domain, ing, customTLS) if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, secret, func(s *corev1.Secret) { // Labels might have changed if the Ingress has been updated to use a // different ProxyGroup. s.Labels = secret.Labels + s.Type = secret.Type + if customTLS != nil { + s.Data = secret.Data + } }); err != nil { return fmt.Errorf("failed to create or update Secret %s: %w", secret.Name, err) } @@ -1057,18 +1075,15 @@ func certSecretRoleBinding(pg *tsapi.ProxyGroup, namespace, domain string) *rbac // certSecret creates a Secret that will store the TLS certificate and private // key for the given domain. Domain must be a valid Kubernetes resource name. -func certSecret(pgName, namespace, domain string, parent client.Object) *corev1.Secret { + +func certSecret(pgName, namespace, domain string, parent client.Object, customTLS *ingressCustomTLS) *corev1.Secret { labels := certResourceLabels(pgName, domain) labels[kubetypes.LabelSecretType] = kubetypes.LabelSecretTypeCerts // Labels that let us identify the Ingress resource lets us reconcile // the Ingress when the TLS Secret is updated (for example, when TLS // certs have been provisioned). - labels[LabelParentType] = strings.ToLower(parent.GetObjectKind().GroupVersionKind().Kind) - labels[LabelParentName] = parent.GetName() - if ns := parent.GetNamespace(); ns != "" { - labels[LabelParentNamespace] = ns - } - return &corev1.Secret{ + mkParentLabels(&labels, parent) + secret := &corev1.Secret{ TypeMeta: metav1.TypeMeta{ APIVersion: "v1", Kind: "Secret", @@ -1084,6 +1099,11 @@ func certSecret(pgName, namespace, domain string, parent client.Object) *corev1. }, Type: corev1.SecretTypeTLS, } + if customTLS != nil { + secret.Data[corev1.TLSCertKey] = append([]byte(nil), customTLS.secret.Data[corev1.TLSCertKey]...) + secret.Data[corev1.TLSPrivateKeyKey] = append([]byte(nil), customTLS.secret.Data[corev1.TLSPrivateKeyKey]...) + } + return secret } func certResourceLabels(pgName, domain string) map[string]string { diff --git a/cmd/k8s-operator/ingress-for-pg_test.go b/cmd/k8s-operator/ingress-for-pg_test.go index 37bd25d5dbc45..e32cd1db2e9cf 100644 --- a/cmd/k8s-operator/ingress-for-pg_test.go +++ b/cmd/k8s-operator/ingress-for-pg_test.go @@ -32,6 +32,7 @@ import ( tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/kubetypes" "tailscale.com/tailcfg" + "tailscale.com/types/ptr" ) func TestIngressPGReconciler(t *testing.T) { @@ -1215,7 +1216,7 @@ func TestIngressPGReconciler_AcceptAppCaps(t *testing.T) { Namespace: "default", UID: types.UID("1234-UID"), Annotations: map[string]string{ - "tailscale.com/proxy-group": "test-pg", + "tailscale.com/proxy-group": "test-pg", "tailscale.com/accept-app-caps": "example.com/cap/monitoring,example.com/cap/admin", }, }, @@ -1282,6 +1283,83 @@ func TestIngressPGReconciler_AcceptAppCaps(t *testing.T) { } } +func TestIngressPGReconciler_CustomTLSSecret(t *testing.T) { + ingPGR, fc, ft := setupIngressTest(t) + + backendSvc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "default"}, + Spec: corev1.ServiceSpec{ + ClusterIP: "10.0.0.1", + Ports: []corev1.ServicePort{{Port: 8080}}, + }, + } + mustCreate(t, fc, backendSvc) + mustCreate(t, fc, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "wildcard-cert", Namespace: "default"}, + Type: corev1.SecretTypeTLS, + Data: map[string][]byte{ + corev1.TLSCertKey: []byte("fake-cert"), + corev1.TLSPrivateKeyKey: []byte("fake-key"), + }, + }) + + ing := &networkingv1.Ingress{ + TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ingress", + Namespace: "default", + UID: types.UID("1234-UID"), + Annotations: map[string]string{ + "tailscale.com/proxy-group": "test-pg", + }, + }, + Spec: networkingv1.IngressSpec{ + IngressClassName: ptr.To("tailscale"), + DefaultBackend: &networkingv1.IngressBackend{ + Service: &networkingv1.IngressServiceBackend{ + Name: "test", + Port: networkingv1.ServiceBackendPort{Number: 8080}, + }, + }, + TLS: []networkingv1.IngressTLS{{Hosts: []string{"zerg.zergrush.dev"}, SecretName: "wildcard-cert"}}, + }, + } + mustCreate(t, fc, ing) + + expectReconciled(t, ingPGR, "default", "test-ingress") + verifyTailscaleService(t, ft, "svc:zerg", []string{"tcp:443"}) + verifyTailscaledConfig(t, fc, "test-pg", []string{"svc:zerg"}) + + cm := &corev1.ConfigMap{} + if err := fc.Get(context.Background(), types.NamespacedName{Name: "test-pg-ingress-config", Namespace: "operator-ns"}, cm); err != nil { + t.Fatalf("getting ConfigMap: %v", err) + } + cfg := &ipn.ServeConfig{} + if err := json.Unmarshal(cm.BinaryData[serveConfigKey], cfg); err != nil { + t.Fatalf("unmarshaling serve config: %v", err) + } + svc := cfg.Services[tailcfg.ServiceName("svc:zerg")] + if svc == nil { + t.Fatal("service svc:zerg not found in serve config") + } + if _, ok := svc.Web[ipn.HostPort("zerg.zergrush.dev:443")]; !ok { + t.Fatalf("expected custom HTTPS host in service config, got keys %v", maps.Keys(svc.Web)) + } + + expectedTLSSecret := certSecret("test-pg", "operator-ns", "zerg.zergrush.dev", ing, &ingressCustomTLS{ + host: "zerg.zergrush.dev", + secretName: "wildcard-cert", + secret: &corev1.Secret{Data: map[string][]byte{ + corev1.TLSCertKey: []byte("fake-cert"), + corev1.TLSPrivateKeyKey: []byte("fake-key"), + }}, + }) + expectEqual(t, fc, expectedTLSSecret) + expectEqual(t, fc, certSecretRole("test-pg", "operator-ns", "zerg.zergrush.dev")) + pg := &tsapi.ProxyGroup{ObjectMeta: metav1.ObjectMeta{Name: "test-pg"}} + expectEqual(t, fc, certSecretRoleBinding(pg, "operator-ns", "zerg.zergrush.dev")) +} + func setupIngressTest(t *testing.T) (*HAIngressReconciler, client.Client, *fakeTSClient) { tsIngressClass := &networkingv1.IngressClass{ ObjectMeta: metav1.ObjectMeta{Name: "tailscale"}, diff --git a/cmd/k8s-operator/ingress.go b/cmd/k8s-operator/ingress.go index a3078dcb48f3a..887d4f553a555 100644 --- a/cmd/k8s-operator/ingress.go +++ b/cmd/k8s-operator/ingress.go @@ -164,13 +164,22 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga gaugeIngressResources.Set(int64(a.managedIngresses.Len())) a.mu.Unlock() - if !IsHTTPSEnabledOnTailnet(a.ssr.tsnetServer) { + customTLS, err := customTLSForIngress(ctx, a.Client, ing) + if err != nil { + return fmt.Errorf("failed to configure custom TLS for ingress: %w", err) + } + + if customTLS == nil && !IsHTTPSEnabledOnTailnet(a.ssr.tsnetServer) { a.recorder.Event(ing, corev1.EventTypeWarning, "HTTPSNotEnabled", "HTTPS is not enabled on the tailnet; ingress may not work") } // magic443 is a fake hostname that we can use to tell containerboot to swap // out with the real hostname once it's known. - const magic443 = "${TS_CERT_DOMAIN}:443" + httpsEndpoint := "${TS_CERT_DOMAIN}" + if customTLS != nil { + httpsEndpoint = customTLS.host + } + host443 := ipn.HostPort(httpsEndpoint + ":443") sc := &ipn.ServeConfig{ TCP: map[uint16]*ipn.TCPPortHandler{ 443: { @@ -178,18 +187,18 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga }, }, Web: map[ipn.HostPort]*ipn.WebServerConfig{ - magic443: { + host443: { Handlers: map[string]*ipn.HTTPHandler{}, }, }, } if opt.Bool(ing.Annotations[AnnotationFunnel]).EqualBool(true) { sc.AllowFunnel = map[ipn.HostPort]bool{ - magic443: true, + host443: true, } } - web := sc.Web[magic443] + web := sc.Web[host443] var tlsHost string // hostname or FQDN or empty if ing.Spec.TLS != nil && len(ing.Spec.TLS) > 0 && len(ing.Spec.TLS[0].Hosts) > 0 { @@ -208,15 +217,15 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga if isHTTPRedirectEnabled(ing) { logger.Infof("HTTP redirect enabled, setting up port 80 redirect handlers") - const magic80 = "${TS_CERT_DOMAIN}:80" + host80 := ipn.HostPort(httpsEndpoint + ":80") sc.TCP[80] = &ipn.TCPPortHandler{HTTP: true} - sc.Web[magic80] = &ipn.WebServerConfig{ + sc.Web[host80] = &ipn.WebServerConfig{ Handlers: map[string]*ipn.HTTPHandler{}, } - if sc.AllowFunnel != nil && sc.AllowFunnel[magic443] { - sc.AllowFunnel[magic80] = true + if sc.AllowFunnel != nil && sc.AllowFunnel[host443] { + sc.AllowFunnel[host80] = true } - web80 := sc.Web[magic80] + web80 := sc.Web[host80] for mountPoint := range handlers { // We send a 301 - Moved Permanently redirect from HTTP to HTTPS redirectURL := "301:https://${HOST}${REQUEST_URI}" @@ -228,6 +237,11 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga } crl := childResourceLabels(ing.Name, ing.Namespace, "ingress") + if customTLS != nil { + if err := ensureManagedTLSSecret(ctx, a.Client, customTLS.host, a.ssr.operatorNamespace, crl, customTLS.secret); err != nil { + return fmt.Errorf("failed to ensure managed custom TLS Secret: %w", err) + } + } var tags []string if tstr, ok := ing.Annotations[AnnotationTags]; ok { tags = strings.Split(tstr, ",") @@ -245,6 +259,7 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga ProxyClassName: proxyClass, proxyType: proxyTypeIngressResource, LoginServer: a.ssr.loginServer, + CertShareMode: ingressCertShareMode(customTLS != nil), } if val := ing.GetAnnotations()[AnnotationExperimentalForwardClusterTrafficViaL7IngresProxy]; val == "true" { @@ -259,19 +274,28 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga if err != nil { return fmt.Errorf("failed to retrieve Ingress HTTPS endpoint status: %w", err) } + hasHTTPS := customTLS == nil + if customTLS != nil { + hasHTTPS = true + } ing.Status.LoadBalancer.Ingress = nil for _, dev := range devices { - if dev.ingressDNSName == "" { + if dev.ingressDNSName == "" && customTLS == nil { continue } - logger.Debugf("setting Ingress hostname to %q", dev.ingressDNSName) - ports := []networkingv1.IngressPortStatus{ - { + hostname := dev.ingressDNSName + if customTLS != nil { + hostname = customTLS.host + } + logger.Debugf("setting Ingress hostname to %q", hostname) + ports := []networkingv1.IngressPortStatus{} + if hasHTTPS { + ports = append(ports, networkingv1.IngressPortStatus{ Protocol: "TCP", Port: 443, - }, + }) } if isHTTPRedirectEnabled(ing) { ports = append(ports, networkingv1.IngressPortStatus{ @@ -280,7 +304,7 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga }) } ing.Status.LoadBalancer.Ingress = append(ing.Status.LoadBalancer.Ingress, networkingv1.IngressLoadBalancerIngress{ - Hostname: dev.ingressDNSName, + Hostname: hostname, Ports: ports, }) } diff --git a/cmd/k8s-operator/ingress_test.go b/cmd/k8s-operator/ingress_test.go index 0609cdbef9ea5..1c983e7b0bfe0 100644 --- a/cmd/k8s-operator/ingress_test.go +++ b/cmd/k8s-operator/ingress_test.go @@ -8,6 +8,7 @@ package main import ( "context" "reflect" + "strings" "testing" "go.uber.org/zap" @@ -26,6 +27,7 @@ import ( "tailscale.com/kube/kubetypes" "tailscale.com/tailcfg" "tailscale.com/tstest" + "tailscale.com/types/ptr" "tailscale.com/util/mak" ) @@ -57,7 +59,21 @@ func TestTailscaleIngress(t *testing.T) { expectReconciled(t, ingR, "default", "test") - fullName, shortName := findGenName(t, fc, "default", "test", "ingress") + secretList := &corev1.SecretList{} + if err := fc.List(t.Context(), secretList, client.InNamespace("operator-ns"), client.MatchingLabels(childResourceLabels("test", "default", "ingress"))); err != nil { + t.Fatalf("listing generated secrets: %v", err) + } + fullName := "" + for _, secret := range secretList.Items { + if strings.HasSuffix(secret.Name, "-0") { + fullName = secret.Name + break + } + } + if fullName == "" { + t.Fatalf("failed to find generated state Secret among %v", secretList.Items) + } + shortName := strings.TrimSuffix(fullName, "-0") opts := configOpts{ replicas: new(int32(1)), stsName: shortName, @@ -286,7 +302,21 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) { expectReconciled(t, ingR, "default", "test") - fullName, shortName := findGenName(t, fc, "default", "test", "ingress") + secretList := &corev1.SecretList{} + if err := fc.List(t.Context(), secretList, client.InNamespace("operator-ns"), client.MatchingLabels(childResourceLabels("test", "default", "ingress"))); err != nil { + t.Fatalf("listing generated secrets: %v", err) + } + fullName := "" + for _, secret := range secretList.Items { + if strings.HasSuffix(secret.Name, "-0") { + fullName = secret.Name + break + } + } + if fullName == "" { + t.Fatalf("failed to find generated state Secret among %v", secretList.Items) + } + shortName := strings.TrimSuffix(fullName, "-0") opts := configOpts{ stsName: shortName, secretName: fullName, @@ -388,7 +418,21 @@ func TestTailscaleIngressWithServiceMonitor(t *testing.T) { logger: zl.Sugar(), } expectReconciled(t, ingR, "default", "test") - fullName, shortName := findGenName(t, fc, "default", "test", "ingress") + secretList := &corev1.SecretList{} + if err := fc.List(t.Context(), secretList, client.InNamespace("operator-ns"), client.MatchingLabels(childResourceLabels("test", "default", "ingress"))); err != nil { + t.Fatalf("listing generated secrets: %v", err) + } + fullName := "" + for _, secret := range secretList.Items { + if strings.HasSuffix(secret.Name, "-0") { + fullName = secret.Name + break + } + } + if fullName == "" { + t.Fatalf("failed to find generated state Secret among %v", secretList.Items) + } + shortName := strings.TrimSuffix(fullName, "-0") opts := configOpts{ stsName: shortName, secretName: fullName, @@ -871,7 +915,21 @@ func TestTailscaleIngressWithHTTPRedirect(t *testing.T) { expectReconciled(t, ingR, "default", "test") - fullName, shortName := findGenName(t, fc, "default", "test", "ingress") + secretList := &corev1.SecretList{} + if err := fc.List(t.Context(), secretList, client.InNamespace("operator-ns"), client.MatchingLabels(childResourceLabels("test", "default", "ingress"))); err != nil { + t.Fatalf("listing generated secrets: %v", err) + } + fullName := "" + for _, secret := range secretList.Items { + if strings.HasSuffix(secret.Name, "-0") { + fullName = secret.Name + break + } + } + if fullName == "" { + t.Fatalf("failed to find generated state Secret among %v", secretList.Items) + } + shortName := strings.TrimSuffix(fullName, "-0") opts := configOpts{ replicas: new(int32(1)), stsName: shortName, @@ -999,6 +1057,101 @@ func TestTailscaleIngressWithAcceptAppCaps(t *testing.T) { expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeResourceReqs) } +func TestTailscaleIngressWithCustomTLSSecret(t *testing.T) { + fc := fake.NewFakeClient(ingressClass()) + ft := &fakeTSClient{} + fakeTsnetServer := &fakeTSNetServer{} + zl, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } + ingR := &IngressReconciler{ + Client: fc, + ingressClassName: "tailscale", + ssr: &tailscaleSTSReconciler{ + Client: fc, + tsClient: ft, + tsnetServer: fakeTsnetServer, + defaultTags: []string{"tag:k8s"}, + operatorNamespace: "operator-ns", + proxyImage: "tailscale/tailscale", + }, + logger: zl.Sugar(), + } + + ing := ingress() + ing.Spec.TLS = []networkingv1.IngressTLS{{Hosts: []string{"zerg.zergrush.dev"}, SecretName: "wildcard-cert"}} + srcTLS := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "wildcard-cert", Namespace: "default"}, + Type: corev1.SecretTypeTLS, + Data: map[string][]byte{ + corev1.TLSCertKey: []byte("fake-cert"), + corev1.TLSPrivateKeyKey: []byte("fake-key"), + }, + } + mustCreate(t, fc, ing) + mustCreate(t, fc, service()) + mustCreate(t, fc, srcTLS) + + expectReconciled(t, ingR, "default", "test") + + secretList := &corev1.SecretList{} + if err := fc.List(t.Context(), secretList, client.InNamespace("operator-ns"), client.MatchingLabels(childResourceLabels("test", "default", "ingress"))); err != nil { + t.Fatalf("listing generated secrets: %v", err) + } + fullName := "" + for _, secret := range secretList.Items { + if strings.HasSuffix(secret.Name, "-0") { + fullName = secret.Name + break + } + } + if fullName == "" { + t.Fatalf("failed to find generated state Secret among %v", secretList.Items) + } + shortName := strings.TrimSuffix(fullName, "-0") + opts := configOpts{ + replicas: ptr.To[int32](1), + stsName: shortName, + secretName: fullName, + namespace: "default", + parentType: "ingress", + hostname: "zerg", + app: kubetypes.AppIngressResource, + certShareMode: "rw", + serveConfig: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "zerg.zergrush.dev:443": {Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://1.2.3.4:8080/"}, + }}, + }, + }, + } + + expectEqual(t, fc, expectedSecret(t, fc, opts)) + expectEqual(t, fc, expectedHeadlessService(shortName, "ingress")) + expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeResourceReqs) + expectEqual(t, fc, managedTLSSecret("zerg.zergrush.dev", "operator-ns", childResourceLabels("test", "default", "ingress"), srcTLS)) + + mustUpdate(t, fc, "operator-ns", fullName, func(secret *corev1.Secret) { + mak.Set(&secret.Data, "device_id", []byte("1234")) + mak.Set(&secret.Data, "device_fqdn", []byte("zerg.tailnetxyz.ts.net")) + }) + expectReconciled(t, ingR, "default", "test") + + expectedIngress := ingress() + expectedIngress.Spec.TLS = []networkingv1.IngressTLS{{Hosts: []string{"zerg.zergrush.dev"}, SecretName: "wildcard-cert"}} + expectedIngress.Finalizers = append(expectedIngress.Finalizers, "tailscale.com/finalizer") + expectedIngress.Status.LoadBalancer = networkingv1.IngressLoadBalancerStatus{ + Ingress: []networkingv1.IngressLoadBalancerIngress{{ + Hostname: "zerg.zergrush.dev", + Ports: []networkingv1.IngressPortStatus{{Port: 443, Protocol: "TCP"}}, + }}, + } + expectEqual(t, fc, expectedIngress) +} + func TestParseAcceptAppCaps(t *testing.T) { tests := []struct { name string diff --git a/cmd/k8s-operator/ingress_tls.go b/cmd/k8s-operator/ingress_tls.go new file mode 100644 index 0000000000000..eec1e956addcc --- /dev/null +++ b/cmd/k8s-operator/ingress_tls.go @@ -0,0 +1,239 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package main + +import ( + "context" + "fmt" + "strings" + + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "tailscale.com/kube/kubetypes" + "tailscale.com/util/mak" +) + +const indexIngressTLSSecret = ".spec.tls.secretName" + +type ingressCustomTLS struct { + host string + secretName string + secret *corev1.Secret +} + +func customTLSForIngress(ctx context.Context, cl client.Client, ing *networkingv1.Ingress) (*ingressCustomTLS, error) { + host := ingressTLSHost(ing) + if host == "" || len(ing.Spec.TLS) == 0 || ing.Spec.TLS[0].SecretName == "" { + return nil, nil + } + + secret := &corev1.Secret{} + if err := cl.Get(ctx, client.ObjectKey{Namespace: ing.Namespace, Name: ing.Spec.TLS[0].SecretName}, secret); err != nil { + return nil, fmt.Errorf("getting TLS Secret %s/%s: %w", ing.Namespace, ing.Spec.TLS[0].SecretName, err) + } + if len(secret.Data[corev1.TLSCertKey]) == 0 || len(secret.Data[corev1.TLSPrivateKeyKey]) == 0 { + return nil, fmt.Errorf("TLS Secret %s/%s must contain tls.crt and tls.key data", ing.Namespace, ing.Spec.TLS[0].SecretName) + } + + return &ingressCustomTLS{ + host: host, + secretName: ing.Spec.TLS[0].SecretName, + secret: secret, + }, nil +} + +func ingressTLSHost(ing *networkingv1.Ingress) string { + if ing.Spec.TLS != nil && len(ing.Spec.TLS) > 0 && len(ing.Spec.TLS[0].Hosts) > 0 { + return ing.Spec.TLS[0].Hosts[0] + } + return "" +} + +func ingressHTTPSHost(ing *networkingv1.Ingress, defaultHost string) string { + if host := ingressTLSHost(ing); host != "" { + return host + } + return defaultHost +} + +func managedTLSSecret(name, namespace string, labels map[string]string, source *corev1.Secret) *corev1.Secret { + managedLabels := make(map[string]string, len(labels)+1) + for key, value := range labels { + managedLabels[key] = value + } + managedLabels[kubetypes.LabelSecretType] = kubetypes.LabelSecretTypeCerts + + managed := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: managedLabels, + }, + Type: corev1.SecretTypeTLS, + } + if source != nil { + managed.Data = map[string][]byte{ + corev1.TLSCertKey: append([]byte(nil), source.Data[corev1.TLSCertKey]...), + corev1.TLSPrivateKeyKey: append([]byte(nil), source.Data[corev1.TLSPrivateKeyKey]...), + } + } + return managed +} + +func ingressCertShareMode(customTLS bool) string { + if customTLS { + return "rw" + } + return "" +} + +func hasTLSSecretData(ctx context.Context, cl client.Client, ns, name string) (bool, error) { + secret := &corev1.Secret{} + err := cl.Get(ctx, client.ObjectKey{Namespace: ns, Name: name}, secret) + if err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } + return len(secret.Data[corev1.TLSCertKey]) > 0 && len(secret.Data[corev1.TLSPrivateKeyKey]) > 0, nil +} + +func ensureManagedTLSSecret(ctx context.Context, cl client.Client, name, namespace string, labels map[string]string, source *corev1.Secret) error { + secret := managedTLSSecret(name, namespace, labels, source) + _, err := createOrUpdate(ctx, cl, namespace, secret, func(existing *corev1.Secret) { + existing.Labels = secret.Labels + existing.Type = secret.Type + existing.Data = secret.Data + }) + if err != nil { + return fmt.Errorf("creating or updating managed TLS Secret %s/%s: %w", namespace, name, err) + } + return nil +} + +func customTLSSecretsForProxyGroup(ctx context.Context, cl client.Client, pgName string) ([]ingressCustomTLS, error) { + ingList := &networkingv1.IngressList{} + if err := cl.List(ctx, ingList); err != nil { + return nil, fmt.Errorf("listing Ingresses for ProxyGroup %q: %w", pgName, err) + } + + custom := make([]ingressCustomTLS, 0) + for i := range ingList.Items { + ing := &ingList.Items[i] + if ing.Annotations[AnnotationProxyGroup] != pgName { + continue + } + tlsCfg, err := customTLSForIngress(ctx, cl, ing) + if err != nil { + return nil, fmt.Errorf("Ingress %s/%s: %w", ing.Namespace, ing.Name, err) + } + if tlsCfg == nil { + continue + } + custom = append(custom, *tlsCfg) + } + return custom, nil +} + +func proxyGroupUsesCustomTLS(ctx context.Context, cl client.Client, pgName string) (bool, error) { + ingList := &networkingv1.IngressList{} + if err := cl.List(ctx, ingList); err != nil { + return false, fmt.Errorf("listing Ingresses for ProxyGroup %q: %w", pgName, err) + } + + var total, custom int + for i := range ingList.Items { + ing := &ingList.Items[i] + if ing.Annotations[AnnotationProxyGroup] != pgName { + continue + } + total++ + if len(ing.Spec.TLS) > 0 && ing.Spec.TLS[0].SecretName != "" { + custom++ + } + } + + if custom == 0 { + return false, nil + } + if custom != total { + return false, fmt.Errorf("all Ingresses on ProxyGroup %q must set spec.tls[0].secretName when any of them use a custom TLS Secret", pgName) + } + return true, nil +} + +func indexTLSSecretName(o client.Object) []string { + ing, ok := o.(*networkingv1.Ingress) + if !ok || len(ing.Spec.TLS) == 0 { + return nil + } + name := strings.TrimSpace(ing.Spec.TLS[0].SecretName) + if name == "" { + return nil + } + return []string{name} +} + +func ingressesFromTLSSecret(cl client.Client, logger clientLogger, ingressClassName string, requireProxyGroup bool) handler.MapFunc { + return func(ctx context.Context, o client.Object) []reconcile.Request { + secret, ok := o.(*corev1.Secret) + if !ok { + logger.Infof("[unexpected] TLS Secret handler triggered for a non-Secret object") + return nil + } + + ingList := &networkingv1.IngressList{} + if err := cl.List(ctx, ingList, client.InNamespace(secret.Namespace), client.MatchingFields{indexIngressTLSSecret: secret.Name}); err != nil { + logger.Infof("error listing Ingresses for TLS Secret %s/%s: %v", secret.Namespace, secret.Name, err) + return nil + } + + requests := make([]reconcile.Request, 0, len(ingList.Items)) + for _, ing := range ingList.Items { + if ing.Spec.IngressClassName == nil || *ing.Spec.IngressClassName != ingressClassName { + continue + } + hasProxyGroup := ing.Annotations[AnnotationProxyGroup] != "" + if hasProxyGroup != requireProxyGroup { + continue + } + requests = append(requests, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&ing)}) + } + return requests + } +} + +func markManagedTLSSecretLabels(labels map[string]string, parent client.Object) map[string]string { + out := make(map[string]string, len(labels)+3) + for key, value := range labels { + out[key] = value + } + mkParentLabels(&out, parent) + return out +} + +func mkParentLabels(labels *map[string]string, parent client.Object) { + mak.Set(labels, LabelParentType, strings.ToLower(parent.GetObjectKind().GroupVersionKind().Kind)) + mak.Set(labels, LabelParentName, parent.GetName()) + if ns := parent.GetNamespace(); ns != "" { + mak.Set(labels, LabelParentNamespace, ns) + } +} + +type clientLogger interface { + Infof(string, ...any) +} diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index d353c53337fd6..4863dbc1b37f8 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -425,6 +425,7 @@ func runReconcilers(opts reconcilerOpts) { Named("ingress-reconciler"). Watches(&appsv1.StatefulSet{}, ingressChildFilter). Watches(&corev1.Secret{}, ingressChildFilter). + Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(ingressesFromTLSSecret(mgr.GetClient(), startlog, opts.ingressClassName, false))). Watches(&corev1.Service{}, svcHandlerForIngress). Watches(&tsapi.ProxyClass{}, proxyClassFilterForIngress). Complete(&IngressReconciler{ @@ -441,6 +442,9 @@ func runReconcilers(opts reconcilerOpts) { if err := mgr.GetFieldIndexer().IndexField(context.Background(), new(networkingv1.Ingress), indexIngressProxyClass, indexProxyClass); err != nil { startlog.Fatalf("failed setting up ProxyClass indexer for Ingresses: %v", err) } + if err := mgr.GetFieldIndexer().IndexField(context.Background(), new(networkingv1.Ingress), indexIngressTLSSecret, indexTLSSecretName); err != nil { + startlog.Fatalf("failed setting up TLS Secret indexer for Ingresses: %v", err) + } lc, err := opts.tsServer.LocalClient() if err != nil { @@ -457,6 +461,7 @@ func runReconcilers(opts reconcilerOpts) { Named("ingress-pg-reconciler"). Watches(&corev1.Service{}, handler.EnqueueRequestsFromMapFunc(serviceHandlerForIngressPG(mgr.GetClient(), startlog, opts.ingressClassName))). Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(HAIngressesFromSecret(mgr.GetClient(), startlog))). + Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(ingressesFromTLSSecret(mgr.GetClient(), startlog, opts.ingressClassName, true))). Watches(&tsapi.ProxyGroup{}, ingressProxyGroupFilter). Complete(&HAIngressReconciler{ recorder: eventRecorder, diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index 4d5a795d79796..1fd6331f476eb 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -396,11 +396,16 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, tailscaleClie } } + customTLS, err := proxyGroupUsesCustomTLS(ctx, r.Client, pg.Name) + if err != nil { + return r.notReadyErrf(pg, logger, "error determining custom TLS mode: %w", err) + } + defaultImage := r.tsProxyImage if pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer { defaultImage = r.k8sProxyImage } - ss, err := pgStatefulSet(pg, r.tsNamespace, defaultImage, r.tsFirewallMode, tailscaledPort, proxyClass) + ss, err := pgStatefulSet(pg, r.tsNamespace, defaultImage, r.tsFirewallMode, tailscaledPort, proxyClass, customTLS) if err != nil { return r.notReadyErrf(pg, logger, "error generating StatefulSet spec: %w", err) } diff --git a/cmd/k8s-operator/proxygroup_specs.go b/cmd/k8s-operator/proxygroup_specs.go index 60b4bddd5613c..5e7be6ecfd10a 100644 --- a/cmd/k8s-operator/proxygroup_specs.go +++ b/cmd/k8s-operator/proxygroup_specs.go @@ -64,7 +64,7 @@ func pgNodePortService(pg *tsapi.ProxyGroup, name string, namespace string) *cor // Returns the base StatefulSet definition for a ProxyGroup. A ProxyClass may be // applied over the top after. -func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string, port *uint16, proxyClass *tsapi.ProxyClass) (*appsv1.StatefulSet, error) { +func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string, port *uint16, proxyClass *tsapi.ProxyClass, customTLS bool) (*appsv1.StatefulSet, error) { if pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer { return kubeAPIServerStatefulSet(pg, namespace, image, port) } @@ -243,14 +243,21 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string Name: "TS_SERVE_CONFIG", Value: fmt.Sprintf("/etc/proxies/%s", serveConfigKey), }, - corev1.EnvVar{ + ) + if customTLS { + envs = append(envs, corev1.EnvVar{ + Name: "TS_CERT_SHARE_MODE", + Value: "rw", + }) + } else { + envs = append(envs, corev1.EnvVar{ // Run proxies in cert share mode to // ensure that only one TLS cert is // issued for an HA Ingress. Name: "TS_EXPERIMENTAL_CERT_SHARE", Value: "true", - }, - ) + }) + } } return append(c.Env, envs...) }() diff --git a/cmd/k8s-operator/proxygroup_test.go b/cmd/k8s-operator/proxygroup_test.go index 1a50ee1f05f44..2e79cb78c1934 100644 --- a/cmd/k8s-operator/proxygroup_test.go +++ b/cmd/k8s-operator/proxygroup_test.go @@ -1231,6 +1231,21 @@ func TestProxyGroupTypes(t *testing.T) { } }) + t.Run("ingress_type_custom_tls", func(t *testing.T) { + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{Name: "test-ingress-custom", UID: "test-ingress-custom-uid"}, + Spec: tsapi.ProxyGroupSpec{Type: tsapi.ProxyGroupTypeIngress, Replicas: new(int32(0))}, + } + sts, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto", nil, nil, true) + if err != nil { + t.Fatalf("pgStatefulSet(custom tls) failed: %v", err) + } + verifyEnvVar(t, sts, "TS_INTERNAL_APP", kubetypes.AppProxyGroupIngress) + verifyEnvVar(t, sts, "TS_SERVE_CONFIG", "/etc/proxies/serve-config.json") + verifyEnvVar(t, sts, "TS_CERT_SHARE_MODE", "rw") + verifyEnvVarNotPresent(t, sts, "TS_EXPERIMENTAL_CERT_SHARE") + }) + t.Run("kubernetes_api_server_type", func(t *testing.T) { pg := &tsapi.ProxyGroup{ ObjectMeta: metav1.ObjectMeta{ @@ -1958,7 +1973,7 @@ func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.Prox role := pgRole(pg, tsNamespace) roleBinding := pgRoleBinding(pg, tsNamespace) serviceAccount := pgServiceAccount(pg, tsNamespace) - statefulSet, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto", nil, proxyClass) + statefulSet, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto", nil, proxyClass, false) if err != nil { t.Fatal(err) } diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index f38e43f1ea613..b3f83cf5da8ce 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -69,9 +69,9 @@ const ( AnnotationProxyGroup = "tailscale.com/proxy-group" // Annotations settable by users on ingresses. - AnnotationFunnel = "tailscale.com/funnel" - AnnotationHTTPRedirect = "tailscale.com/http-redirect" - AnnotationAcceptAppCaps = "tailscale.com/accept-app-caps" + AnnotationFunnel = "tailscale.com/funnel" + AnnotationHTTPRedirect = "tailscale.com/http-redirect" + AnnotationAcceptAppCaps = "tailscale.com/accept-app-caps" // If set to true, set up iptables/nftables rules in the proxy forward // cluster traffic to the tailnet IP of that proxy. This can only be set @@ -155,6 +155,11 @@ type tailscaleSTSConfig struct { // ordinal number generated by the StatefulSet. HostnamePrefix string + // CertShareMode, if non-empty, configures tailscaled to use dedicated TLS + // Secrets for HTTPS endpoints. This is used for externally managed custom + // certificates that should be read directly from Kubernetes Secrets. + CertShareMode string + // Tailnet specifies the Tailnet resource to use for producing auth keys. Tailnet string } @@ -503,6 +508,9 @@ func (a *tailscaleSTSReconciler) provisionSecrets(ctx context.Context, tailscale } for _, secret := range secrets.Items { + if secret.Labels[kubetypes.LabelSecretType] == kubetypes.LabelSecretTypeCerts { + continue + } var ordinal int32 if _, err := fmt.Sscanf(secret.Name, hsvc.Name+"-%d", &ordinal); err != nil { return nil, err @@ -723,6 +731,12 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S Value: "true", }, ) + if sts.CertShareMode != "" { + container.Env = append(container.Env, corev1.EnvVar{ + Name: "TS_CERT_SHARE_MODE", + Value: sts.CertShareMode, + }) + } if sts.ForwardClusterTrafficViaL7IngressProxy { container.Env = append(container.Env, corev1.EnvVar{ diff --git a/cmd/k8s-operator/testutils_test.go b/cmd/k8s-operator/testutils_test.go index 36b608ef6f4fd..c36d353889c75 100644 --- a/cmd/k8s-operator/testutils_test.go +++ b/cmd/k8s-operator/testutils_test.go @@ -72,6 +72,7 @@ type configOpts struct { secretExtraData map[string][]byte resourceVersion string replicas *int32 + certShareMode string enableMetrics bool serviceMonitorLabels tsapi.Labels } @@ -180,6 +181,9 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef }) tsContainer.VolumeMounts = append(tsContainer.VolumeMounts, corev1.VolumeMount{Name: "serve-config-0", ReadOnly: true, MountPath: path.Join("/etc/tailscaled", opts.secretName)}) } + if opts.certShareMode != "" { + tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{Name: "TS_CERT_SHARE_MODE", Value: opts.certShareMode}) + } tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{ Name: "TS_INTERNAL_APP", Value: opts.app, @@ -279,21 +283,28 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps if err != nil { t.Fatal(err) } + envs := []corev1.EnvVar{ + {Name: "TS_USERSPACE", Value: "true"}, + {Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "status.podIP"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}}, + {Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.name"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}}, + {Name: "POD_UID", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.uid"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}}, + {Name: "TS_KUBE_SECRET", Value: "$(POD_NAME)"}, + {Name: "TS_EXPERIMENTAL_SERVICE_AUTO_ADVERTISEMENT", Value: "false"}, + {Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig/$(POD_NAME)"}, + {Name: "TS_DEBUG_ACME_FORCE_RENEWAL", Value: "true"}, + } + if opts.certShareMode != "" { + envs = append(envs, corev1.EnvVar{Name: "TS_CERT_SHARE_MODE", Value: opts.certShareMode}) + } + envs = append(envs, + corev1.EnvVar{Name: "TS_SERVE_CONFIG", Value: "/etc/tailscaled/$(POD_NAME)/serve-config"}, + corev1.EnvVar{Name: "TS_INTERNAL_APP", Value: opts.app}, + ) + tsContainer := corev1.Container{ - Name: "tailscale", - Image: "tailscale/tailscale", - Env: []corev1.EnvVar{ - {Name: "TS_USERSPACE", Value: "true"}, - {Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "status.podIP"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}}, - {Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.name"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}}, - {Name: "POD_UID", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.uid"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}}, - {Name: "TS_KUBE_SECRET", Value: "$(POD_NAME)"}, - {Name: "TS_EXPERIMENTAL_SERVICE_AUTO_ADVERTISEMENT", Value: "false"}, - {Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig/$(POD_NAME)"}, - {Name: "TS_DEBUG_ACME_FORCE_RENEWAL", Value: "true"}, - {Name: "TS_SERVE_CONFIG", Value: "/etc/tailscaled/$(POD_NAME)/serve-config"}, - {Name: "TS_INTERNAL_APP", Value: opts.app}, - }, + Name: "tailscale", + Image: "tailscale/tailscale", + Env: envs, ImagePullPolicy: "Always", VolumeMounts: []corev1.VolumeMount{ {Name: "tailscaledconfig-0", ReadOnly: true, MountPath: path.Join("/etc/tsconfig", opts.secretName)}, diff --git a/ipn/ipnlocal/serve.go b/ipn/ipnlocal/serve.go index 9460896ad8d4a..1281ecc161c35 100644 --- a/ipn/ipnlocal/serve.go +++ b/ipn/ipnlocal/serve.go @@ -1297,9 +1297,14 @@ func (b *LocalBackend) webServerConfig(hostname string, forVIPService tailcfg.Se return c, false } if forVIPService != "" { + key := ipn.HostPort(net.JoinHostPort(hostname, fmt.Sprintf("%d", port))) + if cfg, ok := b.serveConfig.FindServiceWeb(forVIPService, key); ok { + return cfg, true + } + magicDNSSuffix := b.currentNode().NetMap().MagicDNSSuffix() fqdn := strings.Join([]string{forVIPService.WithoutPrefix(), magicDNSSuffix}, ".") - key := ipn.HostPort(net.JoinHostPort(fqdn, fmt.Sprintf("%d", port))) + key = ipn.HostPort(net.JoinHostPort(fqdn, fmt.Sprintf("%d", port))) return b.serveConfig.FindServiceWeb(forVIPService, key) } key := ipn.HostPort(net.JoinHostPort(hostname, fmt.Sprintf("%d", port))) diff --git a/ipn/ipnlocal/serve_test.go b/ipn/ipnlocal/serve_test.go index b3f48b105c8f7..f3095d0b39a7f 100644 --- a/ipn/ipnlocal/serve_test.go +++ b/ipn/ipnlocal/serve_test.go @@ -529,6 +529,64 @@ func TestServeConfigServices(t *testing.T) { } } +func TestServeConfigServicesCustomHost(t *testing.T) { + b := newTestBackend(t) + svcIPMapJSON, err := json.Marshal(tailcfg.ServiceIPMappings{ + "svc:foo": {netip.MustParseAddr("100.101.101.101")}, + }) + if err != nil { + t.Fatal(err) + } + b.currentNode().SetNetMap(&netmap.NetworkMap{ + SelfNode: (&tailcfg.Node{ + Name: "example.ts.net", + CapMap: tailcfg.NodeCapMap{ + tailcfg.NodeAttrServiceHost: []tailcfg.RawMessage{tailcfg.RawMessage(svcIPMapJSON)}, + }, + }).View(), + }) + + conf := &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:foo": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 443: {HTTPS: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "foo.example.com:443": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Text: "ok"}, + }, + }, + }, + }, + }, + } + if err := b.SetServeConfig(conf, ""); err != nil { + t.Fatal(err) + } + + req := &http.Request{ + Host: "foo.example.com", + URL: &url.URL{Path: "/"}, + TLS: &tls.ConnectionState{ServerName: "foo.example.com"}, + } + req = req.WithContext(serveHTTPContextKey.WithValue(req.Context(), &serveHTTPContext{ + ForVIPService: "svc:foo", + DestPort: 443, + SrcAddr: netip.MustParseAddrPort("100.64.0.1:12345"), + })) + + w := httptest.NewRecorder() + b.serveWebHandler(w, req) + if w.Code != http.StatusOK { + t.Fatalf("got status %d, want %d", w.Code, http.StatusOK) + } + if body := strings.TrimSpace(w.Body.String()); body != "ok" { + t.Fatalf("got body %q, want %q", body, "ok") + } +} + func TestServeConfigETag(t *testing.T) { b := newTestBackend(t) From 0cdbe23c4800c917cd44ae56701d55b2529c9142 Mon Sep 17 00:00:00 2001 From: Ryan Mulligan Date: Wed, 25 Mar 2026 13:16:17 -0700 Subject: [PATCH 03/15] cmd/k8s-operator: keep MagicDNS alongside custom TLS --- cmd/k8s-operator/api-server-proxy-pg.go | 2 +- cmd/k8s-operator/api-server-proxy-pg_test.go | 4 +- cmd/k8s-operator/ingress-for-pg.go | 73 +++++------- cmd/k8s-operator/ingress-for-pg_test.go | 26 +++-- cmd/k8s-operator/ingress.go | 63 ++++------ cmd/k8s-operator/ingress_test.go | 23 ++-- cmd/k8s-operator/ingress_tls.go | 114 ++++--------------- cmd/k8s-operator/proxygroup.go | 7 +- cmd/k8s-operator/proxygroup_specs.go | 15 +-- cmd/k8s-operator/proxygroup_test.go | 17 +-- cmd/k8s-operator/sts.go | 20 ++-- 11 files changed, 126 insertions(+), 238 deletions(-) diff --git a/cmd/k8s-operator/api-server-proxy-pg.go b/cmd/k8s-operator/api-server-proxy-pg.go index db3299bd32333..0900fd0aaa264 100644 --- a/cmd/k8s-operator/api-server-proxy-pg.go +++ b/cmd/k8s-operator/api-server-proxy-pg.go @@ -331,7 +331,7 @@ func (r *KubeAPIServerTSServiceReconciler) deleteFinalizer(ctx context.Context, } func (r *KubeAPIServerTSServiceReconciler) ensureCertResources(ctx context.Context, pg *tsapi.ProxyGroup, domain string) error { - secret := certSecret(pg.Name, r.tsNamespace, domain, pg, nil) + secret := certSecret(pg.Name, r.tsNamespace, domain, pg) if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, secret, func(s *corev1.Secret) { s.Labels = secret.Labels }); err != nil { diff --git a/cmd/k8s-operator/api-server-proxy-pg_test.go b/cmd/k8s-operator/api-server-proxy-pg_test.go index 43df670ad89d7..52dda93e515ee 100644 --- a/cmd/k8s-operator/api-server-proxy-pg_test.go +++ b/cmd/k8s-operator/api-server-proxy-pg_test.go @@ -181,7 +181,7 @@ func TestAPIServerProxyReconciler(t *testing.T) { expectedCfg.APIServerProxy.ServiceName = new(tailcfg.ServiceName("svc:" + pgName)) expectCfg(&expectedCfg) - expectEqual(t, fc, certSecret(pgName, ns, defaultDomain, pg, nil)) + expectEqual(t, fc, certSecret(pgName, ns, defaultDomain, pg)) expectEqual(t, fc, certSecretRole(pgName, ns, defaultDomain)) expectEqual(t, fc, certSecretRoleBinding(pg, ns, defaultDomain)) @@ -243,7 +243,7 @@ func TestAPIServerProxyReconciler(t *testing.T) { pg.Status.URL = "" expectEqual(t, fc, pg, omitPGStatusConditionMessages) - expectEqual(t, fc, certSecret(pgName, ns, updatedDomain, pg, nil)) + expectEqual(t, fc, certSecret(pgName, ns, updatedDomain, pg)) expectEqual(t, fc, certSecretRole(pgName, ns, updatedDomain)) expectEqual(t, fc, certSecretRoleBinding(pg, ns, updatedDomain)) expectMissing[corev1.Secret](t, fc, ns, defaultDomain) diff --git a/cmd/k8s-operator/ingress-for-pg.go b/cmd/k8s-operator/ingress-for-pg.go index acdd29b111594..f36b0259f1091 100644 --- a/cmd/k8s-operator/ingress-for-pg.go +++ b/cmd/k8s-operator/ingress-for-pg.go @@ -258,8 +258,9 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin if customTLS != nil { httpsHost = customTLS.host } + serviceHosts := ingressHTTPSHosts(dnsName, customTLS) - if err = r.ensureCertResources(ctx, pg, httpsHost, ing, customTLS); err != nil { + if err = r.ensureCertResources(ctx, pg, dnsName, ing, customTLS); err != nil { return false, fmt.Errorf("error ensuring cert resources: %w", err) } @@ -272,7 +273,6 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin logger.Infof("no Ingress serve config ConfigMap found, unable to update serve config. Ensure that ProxyGroup is healthy.") return svcsChanged, nil } - ep := ipn.HostPort(fmt.Sprintf("%s:443", httpsHost)) handlers, err := handlersForIngress(ctx, ing, r.Client, r.recorder, httpsHost, logger) if err != nil { return false, fmt.Errorf("failed to get handlers for Ingress: %w", err) @@ -283,40 +283,34 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin HTTPS: true, }, }, - Web: map[ipn.HostPort]*ipn.WebServerConfig{ - ep: { - Handlers: handlers, - }, - }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{}, + } + for _, host := range serviceHosts { + ingCfg.Web[ipn.HostPort(fmt.Sprintf("%s:443", host))] = &ipn.WebServerConfig{Handlers: handlers} } // Add HTTP endpoint if configured. if isHTTPEndpointEnabled(ing) { logger.Infof("exposing Ingress over HTTP") - epHTTP := ipn.HostPort(fmt.Sprintf("%s:80", httpsHost)) ingCfg.TCP[80] = &ipn.TCPPortHandler{ HTTP: true, } - ingCfg.Web[epHTTP] = &ipn.WebServerConfig{ - Handlers: handlers, + for _, host := range serviceHosts { + ingCfg.Web[ipn.HostPort(fmt.Sprintf("%s:80", host))] = &ipn.WebServerConfig{Handlers: handlers} } if isHTTPRedirectEnabled(ing) { logger.Warnf("Both HTTP endpoint and HTTP redirect flags are enabled: ignoring HTTP redirect.") } } else if isHTTPRedirectEnabled(ing) { logger.Infof("HTTP redirect enabled, setting up port 80 redirect handlers") - epHTTP := ipn.HostPort(fmt.Sprintf("%s:80", httpsHost)) ingCfg.TCP[80] = &ipn.TCPPortHandler{HTTP: true} - ingCfg.Web[epHTTP] = &ipn.WebServerConfig{ - Handlers: map[string]*ipn.HTTPHandler{}, - } - web80 := ingCfg.Web[epHTTP] - for mountPoint := range handlers { - // We send a 301 - Moved Permanently redirect from HTTP to HTTPS - redirectURL := "301:https://${HOST}${REQUEST_URI}" - logger.Debugf("Creating redirect handler: %s -> %s", mountPoint, redirectURL) - web80.Handlers[mountPoint] = &ipn.HTTPHandler{ - Redirect: redirectURL, + for _, host := range serviceHosts { + epHTTP := ipn.HostPort(fmt.Sprintf("%s:80", host)) + ingCfg.Web[epHTTP] = &ipn.WebServerConfig{Handlers: map[string]*ipn.HTTPHandler{}} + for mountPoint := range handlers { + redirectURL := "301:https://${HOST}${REQUEST_URI}" + logger.Debugf("Creating redirect handler: %s -> %s", mountPoint, redirectURL) + ingCfg.Web[epHTTP].Handlers[mountPoint] = &ipn.HTTPHandler{Redirect: redirectURL} } } } @@ -378,7 +372,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin if isHTTPEndpointEnabled(ing) || isHTTPRedirectEnabled(ing) { mode = serviceAdvertisementHTTPAndHTTPS } - if err = r.maybeUpdateAdvertiseServicesConfig(ctx, serviceName, mode, pg, httpsHost); err != nil { + if err = r.maybeUpdateAdvertiseServicesConfig(ctx, serviceName, mode, pg, customTLS != nil); err != nil { return false, fmt.Errorf("failed to update tailscaled config: %w", err) } @@ -395,12 +389,11 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin ing.Status.LoadBalancer.Ingress = nil default: var ports []networkingv1.IngressPortStatus - hasCerts, err := hasTLSSecretData(ctx, r.Client, r.tsNamespace, httpsHost) + hasCerts, err := hasCerts(ctx, r.Client, r.tsNamespace, serviceName, pg) if err != nil { return false, fmt.Errorf("error checking TLS credentials provisioned for Ingress: %w", err) } - // If TLS certs have not been issued (yet), do not set port 443. - if hasCerts { + if customTLS != nil || hasCerts { ports = append(ports, networkingv1.IngressPortStatus{ Protocol: "TCP", Port: 443, @@ -493,7 +486,7 @@ func (r *HAIngressReconciler) maybeCleanupProxyGroup(ctx context.Context, logger } // Make sure the Tailscale Service is not advertised in tailscaled or serve config. - if err = r.maybeUpdateAdvertiseServicesConfig(ctx, tsSvcName, serviceAdvertisementOff, pg, ""); err != nil { + if err = r.maybeUpdateAdvertiseServicesConfig(ctx, tsSvcName, serviceAdvertisementOff, pg, false); err != nil { return false, fmt.Errorf("failed to update tailscaled config services: %w", err) } @@ -579,7 +572,7 @@ func (r *HAIngressReconciler) maybeCleanup(ctx context.Context, hostname string, } // 4. Unadvertise the Tailscale Service in tailscaled config. - if err = r.maybeUpdateAdvertiseServicesConfig(ctx, serviceName, serviceAdvertisementOff, pg, ""); err != nil { + if err = r.maybeUpdateAdvertiseServicesConfig(ctx, serviceName, serviceAdvertisementOff, pg, false); err != nil { return false, fmt.Errorf("failed to update tailscaled config services: %w", err) } @@ -774,7 +767,7 @@ const ( serviceAdvertisementHTTPAndHTTPS // Both ports 80 and 443 should be advertised ) -func (r *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Context, serviceName tailcfg.ServiceName, mode serviceAdvertisementMode, pg *tsapi.ProxyGroup, httpsHost string) (err error) { +func (r *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Context, serviceName tailcfg.ServiceName, mode serviceAdvertisementMode, pg *tsapi.ProxyGroup, customTLSReady bool) (err error) { // Get all config Secrets for this ProxyGroup. secrets := &corev1.SecretList{} if err := r.List(ctx, secrets, client.InNamespace(r.tsNamespace), client.MatchingLabels(pgSecretLabels(pg.Name, kubetypes.LabelSecretTypeConfig))); err != nil { @@ -789,18 +782,12 @@ func (r *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con // The only exception is Ingresses with an HTTP endpoint enabled - if an // Ingress has an HTTP endpoint enabled, it will be advertised even if the // TLS cert is not yet provisioned. - if httpsHost == "" { - httpsHost, err = dnsNameForService(ctx, r.Client, serviceName, pg, r.tsNamespace) - if err != nil { - return fmt.Errorf("error determining TLS hostname for service %q: %w", serviceName, err) - } - } - hasCert, err := hasTLSSecretData(ctx, r.Client, r.tsNamespace, httpsHost) + hasCert, err := hasCerts(ctx, r.Client, r.tsNamespace, serviceName, pg) if err != nil { return fmt.Errorf("error checking TLS credentials provisioned for service %q: %w", serviceName, err) } shouldBeAdvertised := (mode == serviceAdvertisementHTTPAndHTTPS) || - (mode == serviceAdvertisementHTTPS && hasCert) // if we only expose port 443 and don't have certs (yet), do not advertise + (mode == serviceAdvertisementHTTPS && (hasCert || customTLSReady)) // if we only expose port 443 and don't have certs (yet), do not advertise for _, secret := range secrets.Items { var updated bool @@ -963,18 +950,18 @@ func ownersAreSetAndEqual(a, b *tailscale.VIPService) bool { // https://github.com/tailscale/tailscale/blob/8b1e7f646ee4730ad06c9b70c13e7861b964949b/util/dnsname/dnsname.go#L99 // https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-subdomain-names func (r *HAIngressReconciler) ensureCertResources(ctx context.Context, pg *tsapi.ProxyGroup, domain string, ing *networkingv1.Ingress, customTLS *ingressCustomTLS) error { - secret := certSecret(pg.Name, r.tsNamespace, domain, ing, customTLS) + secret := certSecret(pg.Name, r.tsNamespace, domain, ing) if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, secret, func(s *corev1.Secret) { // Labels might have changed if the Ingress has been updated to use a // different ProxyGroup. s.Labels = secret.Labels s.Type = secret.Type - if customTLS != nil { - s.Data = secret.Data - } }); err != nil { return fmt.Errorf("failed to create or update Secret %s: %w", secret.Name, err) } + if err := ensureCustomTLSStateSecrets(ctx, r.Client, r.tsNamespace, pg, customTLS); err != nil { + return fmt.Errorf("failed to ensure custom TLS state Secrets: %w", err) + } role := certSecretRole(pg.Name, r.tsNamespace, domain) if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, role, func(r *rbacv1.Role) { // Labels might have changed if the Ingress has been updated to use a @@ -1076,7 +1063,7 @@ func certSecretRoleBinding(pg *tsapi.ProxyGroup, namespace, domain string) *rbac // certSecret creates a Secret that will store the TLS certificate and private // key for the given domain. Domain must be a valid Kubernetes resource name. -func certSecret(pgName, namespace, domain string, parent client.Object, customTLS *ingressCustomTLS) *corev1.Secret { +func certSecret(pgName, namespace, domain string, parent client.Object) *corev1.Secret { labels := certResourceLabels(pgName, domain) labels[kubetypes.LabelSecretType] = kubetypes.LabelSecretTypeCerts // Labels that let us identify the Ingress resource lets us reconcile @@ -1099,10 +1086,6 @@ func certSecret(pgName, namespace, domain string, parent client.Object, customTL }, Type: corev1.SecretTypeTLS, } - if customTLS != nil { - secret.Data[corev1.TLSCertKey] = append([]byte(nil), customTLS.secret.Data[corev1.TLSCertKey]...) - secret.Data[corev1.TLSPrivateKeyKey] = append([]byte(nil), customTLS.secret.Data[corev1.TLSPrivateKeyKey]...) - } return secret } diff --git a/cmd/k8s-operator/ingress-for-pg_test.go b/cmd/k8s-operator/ingress-for-pg_test.go index e32cd1db2e9cf..25d4940ba8c04 100644 --- a/cmd/k8s-operator/ingress-for-pg_test.go +++ b/cmd/k8s-operator/ingress-for-pg_test.go @@ -33,6 +33,7 @@ import ( "tailscale.com/kube/kubetypes" "tailscale.com/tailcfg" "tailscale.com/types/ptr" + "tailscale.com/util/mak" ) func TestIngressPGReconciler(t *testing.T) { @@ -1345,19 +1346,24 @@ func TestIngressPGReconciler_CustomTLSSecret(t *testing.T) { if _, ok := svc.Web[ipn.HostPort("zerg.zergrush.dev:443")]; !ok { t.Fatalf("expected custom HTTPS host in service config, got keys %v", maps.Keys(svc.Web)) } + if _, ok := svc.Web[ipn.HostPort("zerg.ts.net:443")]; !ok { + t.Fatalf("expected MagicDNS HTTPS host in service config, got keys %v", maps.Keys(svc.Web)) + } - expectedTLSSecret := certSecret("test-pg", "operator-ns", "zerg.zergrush.dev", ing, &ingressCustomTLS{ - host: "zerg.zergrush.dev", - secretName: "wildcard-cert", - secret: &corev1.Secret{Data: map[string][]byte{ - corev1.TLSCertKey: []byte("fake-cert"), - corev1.TLSPrivateKeyKey: []byte("fake-key"), - }}, - }) + expectedTLSSecret := certSecret("test-pg", "operator-ns", "zerg.ts.net", ing) expectEqual(t, fc, expectedTLSSecret) - expectEqual(t, fc, certSecretRole("test-pg", "operator-ns", "zerg.zergrush.dev")) + expectEqual(t, fc, certSecretRole("test-pg", "operator-ns", "zerg.ts.net")) pg := &tsapi.ProxyGroup{ObjectMeta: metav1.ObjectMeta{Name: "test-pg"}} - expectEqual(t, fc, certSecretRoleBinding(pg, "operator-ns", "zerg.zergrush.dev")) + expectEqual(t, fc, certSecretRoleBinding(pg, "operator-ns", "zerg.ts.net")) + + stateSecret := &corev1.Secret{} + if err := fc.Get(context.Background(), types.NamespacedName{Name: "test-pg-0", Namespace: "operator-ns"}, stateSecret); err != nil { + t.Fatalf("getting state Secret: %v", err) + } + expectedStateSecret := stateSecret.DeepCopy() + mak.Set(&expectedStateSecret.Data, "zerg.zergrush.dev.crt", []byte("fake-cert")) + mak.Set(&expectedStateSecret.Data, "zerg.zergrush.dev.key", []byte("fake-key")) + expectEqual(t, fc, expectedStateSecret) } func setupIngressTest(t *testing.T) (*HAIngressReconciler, client.Client, *fakeTSClient) { diff --git a/cmd/k8s-operator/ingress.go b/cmd/k8s-operator/ingress.go index 887d4f553a555..67ae14c5822b5 100644 --- a/cmd/k8s-operator/ingress.go +++ b/cmd/k8s-operator/ingress.go @@ -173,33 +173,25 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga a.recorder.Event(ing, corev1.EventTypeWarning, "HTTPSNotEnabled", "HTTPS is not enabled on the tailnet; ingress may not work") } - // magic443 is a fake hostname that we can use to tell containerboot to swap - // out with the real hostname once it's known. - httpsEndpoint := "${TS_CERT_DOMAIN}" - if customTLS != nil { - httpsEndpoint = customTLS.host - } - host443 := ipn.HostPort(httpsEndpoint + ":443") + httpsHosts := ingressHTTPSHosts("${TS_CERT_DOMAIN}", customTLS) sc := &ipn.ServeConfig{ TCP: map[uint16]*ipn.TCPPortHandler{ 443: { HTTPS: true, }, }, - Web: map[ipn.HostPort]*ipn.WebServerConfig{ - host443: { - Handlers: map[string]*ipn.HTTPHandler{}, - }, - }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{}, + } + for _, host := range httpsHosts { + sc.Web[ipn.HostPort(host+":443")] = &ipn.WebServerConfig{Handlers: map[string]*ipn.HTTPHandler{}} } if opt.Bool(ing.Annotations[AnnotationFunnel]).EqualBool(true) { - sc.AllowFunnel = map[ipn.HostPort]bool{ - host443: true, + sc.AllowFunnel = map[ipn.HostPort]bool{} + for _, host := range httpsHosts { + sc.AllowFunnel[ipn.HostPort(host+":443")] = true } } - web := sc.Web[host443] - var tlsHost string // hostname or FQDN or empty if ing.Spec.TLS != nil && len(ing.Spec.TLS) > 0 && len(ing.Spec.TLS[0].Hosts) > 0 { tlsHost = ing.Spec.TLS[0].Hosts[0] @@ -208,40 +200,33 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga if err != nil { return fmt.Errorf("failed to get handlers for ingress: %w", err) } - web.Handlers = handlers - if len(web.Handlers) == 0 { + if len(handlers) == 0 { logger.Warn("Ingress contains no valid backends") a.recorder.Eventf(ing, corev1.EventTypeWarning, "NoValidBackends", "no valid backends") return nil } + for _, host := range httpsHosts { + sc.Web[ipn.HostPort(host+":443")].Handlers = handlers + } if isHTTPRedirectEnabled(ing) { logger.Infof("HTTP redirect enabled, setting up port 80 redirect handlers") - host80 := ipn.HostPort(httpsEndpoint + ":80") sc.TCP[80] = &ipn.TCPPortHandler{HTTP: true} - sc.Web[host80] = &ipn.WebServerConfig{ - Handlers: map[string]*ipn.HTTPHandler{}, - } - if sc.AllowFunnel != nil && sc.AllowFunnel[host443] { - sc.AllowFunnel[host80] = true - } - web80 := sc.Web[host80] - for mountPoint := range handlers { - // We send a 301 - Moved Permanently redirect from HTTP to HTTPS - redirectURL := "301:https://${HOST}${REQUEST_URI}" - logger.Debugf("Creating redirect handler: %s -> %s", mountPoint, redirectURL) - web80.Handlers[mountPoint] = &ipn.HTTPHandler{ - Redirect: redirectURL, + for _, host := range httpsHosts { + host80 := ipn.HostPort(host + ":80") + sc.Web[host80] = &ipn.WebServerConfig{Handlers: map[string]*ipn.HTTPHandler{}} + if sc.AllowFunnel != nil { + sc.AllowFunnel[host80] = true + } + for mountPoint := range handlers { + redirectURL := "301:https://${HOST}${REQUEST_URI}" + logger.Debugf("Creating redirect handler: %s -> %s", mountPoint, redirectURL) + sc.Web[host80].Handlers[mountPoint] = &ipn.HTTPHandler{Redirect: redirectURL} } } } crl := childResourceLabels(ing.Name, ing.Namespace, "ingress") - if customTLS != nil { - if err := ensureManagedTLSSecret(ctx, a.Client, customTLS.host, a.ssr.operatorNamespace, crl, customTLS.secret); err != nil { - return fmt.Errorf("failed to ensure managed custom TLS Secret: %w", err) - } - } var tags []string if tstr, ok := ing.Annotations[AnnotationTags]; ok { tags = strings.Split(tstr, ",") @@ -259,7 +244,9 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga ProxyClassName: proxyClass, proxyType: proxyTypeIngressResource, LoginServer: a.ssr.loginServer, - CertShareMode: ingressCertShareMode(customTLS != nil), + } + if customTLS != nil { + sts.CustomTLSCerts = map[string]*corev1.Secret{customTLS.host: customTLS.secret} } if val := ing.GetAnnotations()[AnnotationExperimentalForwardClusterTrafficViaL7IngresProxy]; val == "true" { diff --git a/cmd/k8s-operator/ingress_test.go b/cmd/k8s-operator/ingress_test.go index 1c983e7b0bfe0..0966cd7b38b8c 100644 --- a/cmd/k8s-operator/ingress_test.go +++ b/cmd/k8s-operator/ingress_test.go @@ -1111,28 +1111,33 @@ func TestTailscaleIngressWithCustomTLSSecret(t *testing.T) { } shortName := strings.TrimSuffix(fullName, "-0") opts := configOpts{ - replicas: ptr.To[int32](1), - stsName: shortName, - secretName: fullName, - namespace: "default", - parentType: "ingress", - hostname: "zerg", - app: kubetypes.AppIngressResource, - certShareMode: "rw", + replicas: ptr.To[int32](1), + stsName: shortName, + secretName: fullName, + namespace: "default", + parentType: "ingress", + hostname: "zerg", + app: kubetypes.AppIngressResource, serveConfig: &ipn.ServeConfig{ TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}}, Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "${TS_CERT_DOMAIN}:443": {Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://1.2.3.4:8080/"}, + }}, "zerg.zergrush.dev:443": {Handlers: map[string]*ipn.HTTPHandler{ "/": {Proxy: "http://1.2.3.4:8080/"}, }}, }, }, + secretExtraData: map[string][]byte{ + "zerg.zergrush.dev.crt": []byte("fake-cert"), + "zerg.zergrush.dev.key": []byte("fake-key"), + }, } expectEqual(t, fc, expectedSecret(t, fc, opts)) expectEqual(t, fc, expectedHeadlessService(shortName, "ingress")) expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeResourceReqs) - expectEqual(t, fc, managedTLSSecret("zerg.zergrush.dev", "operator-ns", childResourceLabels("test", "default", "ingress"), srcTLS)) mustUpdate(t, fc, "operator-ns", fullName, func(secret *corev1.Secret) { mak.Set(&secret.Data, "device_id", []byte("1234")) diff --git a/cmd/k8s-operator/ingress_tls.go b/cmd/k8s-operator/ingress_tls.go index eec1e956addcc..18b423f2babd7 100644 --- a/cmd/k8s-operator/ingress_tls.go +++ b/cmd/k8s-operator/ingress_tls.go @@ -13,11 +13,11 @@ import ( corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/reconcile" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/kubetypes" "tailscale.com/util/mak" ) @@ -65,41 +65,6 @@ func ingressHTTPSHost(ing *networkingv1.Ingress, defaultHost string) string { return defaultHost } -func managedTLSSecret(name, namespace string, labels map[string]string, source *corev1.Secret) *corev1.Secret { - managedLabels := make(map[string]string, len(labels)+1) - for key, value := range labels { - managedLabels[key] = value - } - managedLabels[kubetypes.LabelSecretType] = kubetypes.LabelSecretTypeCerts - - managed := &corev1.Secret{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Secret", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - Labels: managedLabels, - }, - Type: corev1.SecretTypeTLS, - } - if source != nil { - managed.Data = map[string][]byte{ - corev1.TLSCertKey: append([]byte(nil), source.Data[corev1.TLSCertKey]...), - corev1.TLSPrivateKeyKey: append([]byte(nil), source.Data[corev1.TLSPrivateKeyKey]...), - } - } - return managed -} - -func ingressCertShareMode(customTLS bool) string { - if customTLS { - return "rw" - } - return "" -} - func hasTLSSecretData(ctx context.Context, cl client.Client, ns, name string) (bool, error) { secret := &corev1.Secret{} err := cl.Get(ctx, client.ObjectKey{Namespace: ns, Name: name}, secret) @@ -112,68 +77,39 @@ func hasTLSSecretData(ctx context.Context, cl client.Client, ns, name string) (b return len(secret.Data[corev1.TLSCertKey]) > 0 && len(secret.Data[corev1.TLSPrivateKeyKey]) > 0, nil } -func ensureManagedTLSSecret(ctx context.Context, cl client.Client, name, namespace string, labels map[string]string, source *corev1.Secret) error { - secret := managedTLSSecret(name, namespace, labels, source) - _, err := createOrUpdate(ctx, cl, namespace, secret, func(existing *corev1.Secret) { - existing.Labels = secret.Labels - existing.Type = secret.Type - existing.Data = secret.Data - }) - if err != nil { - return fmt.Errorf("creating or updating managed TLS Secret %s/%s: %w", namespace, name, err) +func ingressHTTPSHosts(defaultHost string, customTLS *ingressCustomTLS) []string { + hosts := []string{defaultHost} + if customTLS != nil && customTLS.host != defaultHost { + hosts = append([]string{customTLS.host}, hosts...) } - return nil + return hosts } -func customTLSSecretsForProxyGroup(ctx context.Context, cl client.Client, pgName string) ([]ingressCustomTLS, error) { - ingList := &networkingv1.IngressList{} - if err := cl.List(ctx, ingList); err != nil { - return nil, fmt.Errorf("listing Ingresses for ProxyGroup %q: %w", pgName, err) - } - - custom := make([]ingressCustomTLS, 0) - for i := range ingList.Items { - ing := &ingList.Items[i] - if ing.Annotations[AnnotationProxyGroup] != pgName { - continue - } - tlsCfg, err := customTLSForIngress(ctx, cl, ing) - if err != nil { - return nil, fmt.Errorf("Ingress %s/%s: %w", ing.Namespace, ing.Name, err) - } - if tlsCfg == nil { - continue - } - custom = append(custom, *tlsCfg) +func copyCustomTLSSecretData(data map[string][]byte, customTLS *ingressCustomTLS) { + if customTLS == nil { + return } - return custom, nil + mak.Set(&data, customTLS.host+".crt", append([]byte(nil), customTLS.secret.Data[corev1.TLSCertKey]...)) + mak.Set(&data, customTLS.host+".key", append([]byte(nil), customTLS.secret.Data[corev1.TLSPrivateKeyKey]...)) } -func proxyGroupUsesCustomTLS(ctx context.Context, cl client.Client, pgName string) (bool, error) { - ingList := &networkingv1.IngressList{} - if err := cl.List(ctx, ingList); err != nil { - return false, fmt.Errorf("listing Ingresses for ProxyGroup %q: %w", pgName, err) - } - - var total, custom int - for i := range ingList.Items { - ing := &ingList.Items[i] - if ing.Annotations[AnnotationProxyGroup] != pgName { - continue - } - total++ - if len(ing.Spec.TLS) > 0 && ing.Spec.TLS[0].SecretName != "" { - custom++ - } +func ensureCustomTLSStateSecrets(ctx context.Context, cl client.Client, namespace string, pg *tsapi.ProxyGroup, customTLS *ingressCustomTLS) error { + if customTLS == nil { + return nil } - - if custom == 0 { - return false, nil + secrets := &corev1.SecretList{} + if err := cl.List(ctx, secrets, client.InNamespace(namespace), client.MatchingLabels(pgSecretLabels(pg.Name, kubetypes.LabelSecretTypeState))); err != nil { + return fmt.Errorf("listing ProxyGroup state Secrets for %q: %w", pg.Name, err) } - if custom != total { - return false, fmt.Errorf("all Ingresses on ProxyGroup %q must set spec.tls[0].secretName when any of them use a custom TLS Secret", pgName) + for i := range secrets.Items { + secret := &secrets.Items[i] + orig := secret.DeepCopy() + copyCustomTLSSecretData(secret.Data, customTLS) + if err := cl.Patch(ctx, secret, client.MergeFrom(orig)); err != nil { + return fmt.Errorf("updating ProxyGroup state Secret %s/%s: %w", namespace, secret.Name, err) + } } - return true, nil + return nil } func indexTLSSecretName(o client.Object) []string { diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index 1fd6331f476eb..4d5a795d79796 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -396,16 +396,11 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, tailscaleClie } } - customTLS, err := proxyGroupUsesCustomTLS(ctx, r.Client, pg.Name) - if err != nil { - return r.notReadyErrf(pg, logger, "error determining custom TLS mode: %w", err) - } - defaultImage := r.tsProxyImage if pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer { defaultImage = r.k8sProxyImage } - ss, err := pgStatefulSet(pg, r.tsNamespace, defaultImage, r.tsFirewallMode, tailscaledPort, proxyClass, customTLS) + ss, err := pgStatefulSet(pg, r.tsNamespace, defaultImage, r.tsFirewallMode, tailscaledPort, proxyClass) if err != nil { return r.notReadyErrf(pg, logger, "error generating StatefulSet spec: %w", err) } diff --git a/cmd/k8s-operator/proxygroup_specs.go b/cmd/k8s-operator/proxygroup_specs.go index 5e7be6ecfd10a..60b4bddd5613c 100644 --- a/cmd/k8s-operator/proxygroup_specs.go +++ b/cmd/k8s-operator/proxygroup_specs.go @@ -64,7 +64,7 @@ func pgNodePortService(pg *tsapi.ProxyGroup, name string, namespace string) *cor // Returns the base StatefulSet definition for a ProxyGroup. A ProxyClass may be // applied over the top after. -func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string, port *uint16, proxyClass *tsapi.ProxyClass, customTLS bool) (*appsv1.StatefulSet, error) { +func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string, port *uint16, proxyClass *tsapi.ProxyClass) (*appsv1.StatefulSet, error) { if pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer { return kubeAPIServerStatefulSet(pg, namespace, image, port) } @@ -243,21 +243,14 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string Name: "TS_SERVE_CONFIG", Value: fmt.Sprintf("/etc/proxies/%s", serveConfigKey), }, - ) - if customTLS { - envs = append(envs, corev1.EnvVar{ - Name: "TS_CERT_SHARE_MODE", - Value: "rw", - }) - } else { - envs = append(envs, corev1.EnvVar{ + corev1.EnvVar{ // Run proxies in cert share mode to // ensure that only one TLS cert is // issued for an HA Ingress. Name: "TS_EXPERIMENTAL_CERT_SHARE", Value: "true", - }) - } + }, + ) } return append(c.Env, envs...) }() diff --git a/cmd/k8s-operator/proxygroup_test.go b/cmd/k8s-operator/proxygroup_test.go index 2e79cb78c1934..1a50ee1f05f44 100644 --- a/cmd/k8s-operator/proxygroup_test.go +++ b/cmd/k8s-operator/proxygroup_test.go @@ -1231,21 +1231,6 @@ func TestProxyGroupTypes(t *testing.T) { } }) - t.Run("ingress_type_custom_tls", func(t *testing.T) { - pg := &tsapi.ProxyGroup{ - ObjectMeta: metav1.ObjectMeta{Name: "test-ingress-custom", UID: "test-ingress-custom-uid"}, - Spec: tsapi.ProxyGroupSpec{Type: tsapi.ProxyGroupTypeIngress, Replicas: new(int32(0))}, - } - sts, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto", nil, nil, true) - if err != nil { - t.Fatalf("pgStatefulSet(custom tls) failed: %v", err) - } - verifyEnvVar(t, sts, "TS_INTERNAL_APP", kubetypes.AppProxyGroupIngress) - verifyEnvVar(t, sts, "TS_SERVE_CONFIG", "/etc/proxies/serve-config.json") - verifyEnvVar(t, sts, "TS_CERT_SHARE_MODE", "rw") - verifyEnvVarNotPresent(t, sts, "TS_EXPERIMENTAL_CERT_SHARE") - }) - t.Run("kubernetes_api_server_type", func(t *testing.T) { pg := &tsapi.ProxyGroup{ ObjectMeta: metav1.ObjectMeta{ @@ -1973,7 +1958,7 @@ func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.Prox role := pgRole(pg, tsNamespace) roleBinding := pgRoleBinding(pg, tsNamespace) serviceAccount := pgServiceAccount(pg, tsNamespace) - statefulSet, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto", nil, proxyClass, false) + statefulSet, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto", nil, proxyClass) if err != nil { t.Fatal(err) } diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index b3f83cf5da8ce..c3a72d652452f 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -155,10 +155,11 @@ type tailscaleSTSConfig struct { // ordinal number generated by the StatefulSet. HostnamePrefix string - // CertShareMode, if non-empty, configures tailscaled to use dedicated TLS - // Secrets for HTTPS endpoints. This is used for externally managed custom - // certificates that should be read directly from Kubernetes Secrets. - CertShareMode string + // CustomTLSCerts are copied into the proxy state Secret as + // .crt/.key entries so the proxy can terminate TLS for + // externally managed custom hostnames while still using Tailscale-managed + // certificates for MagicDNS endpoints. + CustomTLSCerts map[string]*corev1.Secret // Tailnet specifies the Tailnet resource to use for producing auth keys. Tailnet string @@ -486,6 +487,10 @@ func (a *tailscaleSTSReconciler) provisionSecrets(ctx context.Context, tailscale mak.Set(&secret.StringData, "serve-config", string(j)) } + for host, src := range stsC.CustomTLSCerts { + mak.Set(&secret.Data, host+".crt", append([]byte(nil), src.Data[corev1.TLSCertKey]...)) + mak.Set(&secret.Data, host+".key", append([]byte(nil), src.Data[corev1.TLSPrivateKeyKey]...)) + } if orig != nil && !apiequality.Semantic.DeepEqual(latest, orig) { logger.With("config", sanitizeConfig(latestConfig)).Debugf("patching the existing proxy Secret") @@ -731,13 +736,6 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S Value: "true", }, ) - if sts.CertShareMode != "" { - container.Env = append(container.Env, corev1.EnvVar{ - Name: "TS_CERT_SHARE_MODE", - Value: sts.CertShareMode, - }) - } - if sts.ForwardClusterTrafficViaL7IngressProxy { container.Env = append(container.Env, corev1.EnvVar{ Name: "EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS", From c2d9d6d95e6fa15558b8d0bd5cf2a0cfe4e76869 Mon Sep 17 00:00:00 2001 From: Ryan Mulligan Date: Wed, 25 Mar 2026 16:08:44 -0700 Subject: [PATCH 04/15] cmd/k8s-operator: cache custom TLS Secrets across namespaces --- cmd/k8s-operator/operator.go | 40 ++++++++++++++++++++++++++++--- cmd/k8s-operator/operator_test.go | 24 +++++++++++++++++++ 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index 4863dbc1b37f8..e618fc86c60d7 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -100,6 +100,7 @@ func main() { isDefaultLoadBalancer = defaultBool("OPERATOR_DEFAULT_LOAD_BALANCER", false) loginServer = strings.TrimSuffix(defaultEnv("OPERATOR_LOGIN_SERVER", ""), "/") ingressClassName = defaultEnv("OPERATOR_INGRESS_CLASS_NAME", "tailscale") + secretNamespaces = splitNamespaces(defaultEnv("OPERATOR_SECRET_NAMESPACES", "")) ) var opts []kzap.Opts @@ -160,6 +161,7 @@ func main() { tsServer: s, tsClient: tsc, tailscaleNamespace: tsNamespace, + secretNamespaces: secretNamespaces, restConfig: restConfig, proxyImage: image, k8sProxyImage: k8sProxyImage, @@ -290,8 +292,8 @@ func serviceManagedResourceFilterPredicate() predicate.Predicate { // ServiceReconciler. It blocks forever. func runReconcilers(opts reconcilerOpts) { startlog := opts.log.Named("startReconcilers") - // For secrets and statefulsets, we only get permission to touch the objects - // in the controller's own namespace. This cannot be expressed by + // For most namespaced resources, we only get permission to touch the + // objects in the controller's own namespace. This cannot be expressed by // .Watches(...) below, instead you have to add a per-type field selector to // the cache that sits a few layers below the builder stuff, which will // implicitly filter what parts of the world the builder code gets to see at @@ -299,6 +301,10 @@ func runReconcilers(opts reconcilerOpts) { nsFilter := cache.ByObject{ Field: client.InNamespace(opts.tailscaleNamespace).AsSelector(), } + secretNamespaces := watchedSecretNamespaces(opts.tailscaleNamespace, opts.secretNamespaces) + secretFilter := cache.ByObject{ + Namespaces: secretNamespaces, + } // We watch the ServiceMonitor CRD to ensure that reconcilers are re-triggered if user's workflows result in the // ServiceMonitor CRD applied after some of our resources that define ServiceMonitor creation. This selector @@ -316,7 +322,7 @@ func runReconcilers(opts reconcilerOpts) { // Other object types (e.g., EndpointSlices) can still be fetched or watched using the cached client, but they will not have any filtering applied. Cache: cache.Options{ ByObject: map[client.Object]cache.ByObject{ - &corev1.Secret{}: nsFilter, + &corev1.Secret{}: secretFilter, &corev1.ServiceAccount{}: nsFilter, &corev1.Pod{}: nsFilter, &corev1.ConfigMap{}: nsFilter, @@ -765,11 +771,39 @@ func runReconcilers(opts reconcilerOpts) { } } +func splitNamespaces(raw string) []string { + if raw == "" { + return nil + } + parts := strings.Split(raw, ",") + namespaces := make([]string, 0, len(parts)) + for _, part := range parts { + part = strings.TrimSpace(part) + if part == "" { + continue + } + namespaces = append(namespaces, part) + } + return namespaces +} + +func watchedSecretNamespaces(operatorNamespace string, extraNamespaces []string) map[string]cache.Config { + namespaces := map[string]cache.Config{operatorNamespace: {}} + for _, ns := range extraNamespaces { + if ns == "" { + continue + } + namespaces[ns] = cache.Config{} + } + return namespaces +} + type reconcilerOpts struct { log *zap.SugaredLogger tsServer *tsnet.Server tsClient tsClient tailscaleNamespace string // namespace in which operator resources will be deployed + secretNamespaces []string // extra namespaces whose Secrets should be cached by the operator restConfig *rest.Config // config for connecting to the kube API server proxyImage string // : k8sProxyImage string // : diff --git a/cmd/k8s-operator/operator_test.go b/cmd/k8s-operator/operator_test.go index 305b1738cbf81..1f0100d2e0411 100644 --- a/cmd/k8s-operator/operator_test.go +++ b/cmd/k8s-operator/operator_test.go @@ -21,6 +21,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -34,6 +35,29 @@ import ( "tailscale.com/util/mak" ) +func TestWatchedSecretNamespaces(t *testing.T) { + t.Run("operator namespace only by default", func(t *testing.T) { + got := watchedSecretNamespaces("tailscale", nil) + want := map[string]cache.Config{"tailscale": {}} + if diff := cmp.Diff(want, got); diff != "" { + t.Fatalf("watchedSecretNamespaces mismatch (-want +got):\n%s", diff) + } + }) + + t.Run("adds trimmed extra namespaces and de-dupes", func(t *testing.T) { + extra := splitNamespaces(" zergrush-system , staging-zergrush-system, tailscale ,, zergrush-system ") + got := watchedSecretNamespaces("tailscale", extra) + want := map[string]cache.Config{ + "tailscale": {}, + "zergrush-system": {}, + "staging-zergrush-system": {}, + } + if diff := cmp.Diff(want, got); diff != "" { + t.Fatalf("watchedSecretNamespaces mismatch (-want +got):\n%s", diff) + } + }) +} + func TestLoadBalancerClass(t *testing.T) { fc := fake.NewFakeClient() ft := &fakeTSClient{} From 2d6c72800584675e7ccf1facf346dfa288664dff Mon Sep 17 00:00:00 2001 From: Ryan Mulligan Date: Wed, 25 Mar 2026 18:33:23 -0700 Subject: [PATCH 05/15] ipn/ipnlocal: serve cached custom domain certs --- ipn/ipnlocal/cert.go | 16 +++++++---- ipn/ipnlocal/cert_test.go | 58 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 6 deletions(-) diff --git a/ipn/ipnlocal/cert.go b/ipn/ipnlocal/cert.go index efab9db7aad6e..a823d798043ee 100644 --- a/ipn/ipnlocal/cert.go +++ b/ipn/ipnlocal/cert.go @@ -125,13 +125,22 @@ func (b *LocalBackend) GetCertPEMWithValidity(ctx context.Context, domain string if !validLookingCertDomain(domain) { return nil, errors.New("invalid domain") } + now := b.clock.Now() + cs, err := b.getCertStore() + if err != nil { + return nil, err + } certDomain, err := b.resolveCertDomain(domain) if err != nil { + if pair, cacheErr := getCertPEMCached(cs, domain, now); cacheErr == nil { + return pair, nil + } else if cacheErr != nil && !errors.Is(cacheErr, ipn.ErrStateNotExist) { + return nil, cacheErr + } return nil, err } logf := logger.WithPrefix(b.logf, fmt.Sprintf("cert(%q): ", domain)) - now := b.clock.Now() traceACME := func(v any) { if !acmeDebug() { return @@ -140,11 +149,6 @@ func (b *LocalBackend) GetCertPEMWithValidity(ctx context.Context, domain string log.Printf("acme %T: %s", v, j) } - cs, err := b.getCertStore() - if err != nil { - return nil, err - } - if pair, err := getCertPEMCached(cs, certDomain, now); err == nil { if envknob.IsCertShareReadOnlyMode() { return pair, nil diff --git a/ipn/ipnlocal/cert_test.go b/ipn/ipnlocal/cert_test.go index cc9146ae1e055..06d4f4b9be4e6 100644 --- a/ipn/ipnlocal/cert_test.go +++ b/ipn/ipnlocal/cert_test.go @@ -581,3 +581,61 @@ func TestGetCertPEMWithValidity(t *testing.T) { }) } } + +func TestGetCertPEMWithValidityUsesCachedCustomDomain(t *testing.T) { + const ( + certDomain = "node.ts.net" + customDomain = "example.com" + ) + b := newTestLocalBackend(t) + b.varRoot = t.TempDir() + b.clock = tstest.NewClock(tstest.ClockOpts{Start: time.Date(2023, time.February, 20, 0, 0, 0, 0, time.UTC)}) + testRoot, err := certTestFS.ReadFile("testdata/rootCA.pem") + if err != nil { + t.Fatal(err) + } + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(testRoot) { + t.Fatal("Unable to add test CA to the cert pool") + } + testX509Roots = roots + defer func() { testX509Roots = nil }() + + b.mu.Lock() + b.currentNode().SetNetMap(&netmap.NetworkMap{ + SelfNode: (&tailcfg.Node{}).View(), + DNS: tailcfg.DNSConfig{ + CertDomains: []string{certDomain}, + }, + }) + b.mu.Unlock() + + certDir, err := b.certDir() + if err != nil { + t.Fatalf("certDir error: %v", err) + } + if err := os.WriteFile(filepath.Join(certDir, customDomain+".crt"), must.Get(os.ReadFile("testdata/example.com.pem")), 0644); err != nil { + t.Fatalf("writing cached cert: %v", err) + } + if err := os.WriteFile(filepath.Join(certDir, customDomain+".key"), must.Get(os.ReadFile("testdata/example.com-key.pem")), 0644); err != nil { + t.Fatalf("writing cached key: %v", err) + } + + called := false + getCertPEM = func(ctx context.Context, b *LocalBackend, cs certStore, logf logger.Logf, traceACME func(any), domain string, now time.Time, minValidity time.Duration) (*TLSCertKeyPair, error) { + called = true + return nil, nil + } + defer func() { getCertPEM = nil }() + + pair, err := b.GetCertPEMWithValidity(context.Background(), customDomain, 0) + if err != nil { + t.Fatalf("GetCertPEMWithValidity(%q): %v", customDomain, err) + } + if pair == nil { + t.Fatalf("GetCertPEMWithValidity(%q) returned nil pair", customDomain) + } + if called { + t.Fatalf("GetCertPEMWithValidity(%q) unexpectedly attempted issuance", customDomain) + } +} From c645702da7c142b21bf3716ab0f5b8aea838a724 Mon Sep 17 00:00:00 2001 From: Ryan Mulligan Date: Thu, 26 Mar 2026 15:33:52 -0700 Subject: [PATCH 06/15] ipn/ipnlocal: scope app caps to VIP services --- ipn/ipnlocal/node_backend.go | 28 ++++++++ ipn/ipnlocal/serve.go | 7 +- ipn/ipnlocal/serve_test.go | 133 +++++++++++++++++++++++++++++++++++ 3 files changed, 167 insertions(+), 1 deletion(-) diff --git a/ipn/ipnlocal/node_backend.go b/ipn/ipnlocal/node_backend.go index 75550b3d5d5f7..e026cfa02774e 100644 --- a/ipn/ipnlocal/node_backend.go +++ b/ipn/ipnlocal/node_backend.go @@ -295,6 +295,14 @@ func (nb *nodeBackend) PeerCaps(src netip.Addr) tailcfg.PeerCapMap { return nb.peerCapsLocked(src) } +// peerCapsForService returns the capabilities that remote src IP has to the +// specified VIP service hosted by this node. +func (nb *nodeBackend) peerCapsForService(src netip.Addr, serviceName tailcfg.ServiceName) tailcfg.PeerCapMap { + nb.mu.Lock() + defer nb.mu.Unlock() + return nb.peerCapsForServiceLocked(src, serviceName) +} + func (nb *nodeBackend) peerCapsLocked(src netip.Addr) tailcfg.PeerCapMap { if nb.netMap == nil { return nil @@ -317,6 +325,26 @@ func (nb *nodeBackend) peerCapsLocked(src netip.Addr) tailcfg.PeerCapMap { return nil } +func (nb *nodeBackend) peerCapsForServiceLocked(src netip.Addr, serviceName tailcfg.ServiceName) tailcfg.PeerCapMap { + if nb.netMap == nil || serviceName == "" { + return nil + } + filt := nb.filterAtomic.Load() + if filt == nil { + return nil + } + serviceIPMap := nb.netMap.GetVIPServiceIPMap() + if len(serviceIPMap) == 0 { + return nil + } + for _, dst := range serviceIPMap[serviceName] { + if dst.BitLen() == src.BitLen() { // match on family + return filt.CapsWithValues(src, dst) + } + } + return nil +} + // PeerHasCap reports whether the peer contains the given capability string, // with any value(s). func (nb *nodeBackend) PeerHasCap(peer tailcfg.NodeView, wantCap tailcfg.PeerCapability) bool { diff --git a/ipn/ipnlocal/serve.go b/ipn/ipnlocal/serve.go index 1281ecc161c35..afd10ab2a321a 100644 --- a/ipn/ipnlocal/serve.go +++ b/ipn/ipnlocal/serve.go @@ -1101,7 +1101,12 @@ func (b *LocalBackend) addAppCapabilitiesHeader(r *httputil.ProxyRequest) error if acceptCaps.IsNil() { return nil } - peerCaps := b.PeerCaps(c.SrcAddr.Addr()) + var peerCaps tailcfg.PeerCapMap + if c.ForVIPService != "" { + peerCaps = b.currentNode().peerCapsForService(c.SrcAddr.Addr(), c.ForVIPService) + } else { + peerCaps = b.PeerCaps(c.SrcAddr.Addr()) + } if peerCaps == nil { return nil } diff --git a/ipn/ipnlocal/serve_test.go b/ipn/ipnlocal/serve_test.go index f3095d0b39a7f..72888245c8e1f 100644 --- a/ipn/ipnlocal/serve_test.go +++ b/ipn/ipnlocal/serve_test.go @@ -1009,6 +1009,139 @@ func TestServeHTTPProxyGrantHeader(t *testing.T) { } } +func TestServeHTTPProxyGrantHeaderForVIPService(t *testing.T) { + b := newTestBackend(t) + + svcIPMapJSON, err := json.Marshal(tailcfg.ServiceIPMappings{ + "svc:foo": {netip.MustParseAddr("100.101.101.101")}, + }) + if err != nil { + t.Fatal(err) + } + + nm := b.NetMap() + self := nm.SelfNode.AsStruct() + self.CapMap = tailcfg.NodeCapMap{ + tailcfg.NodeAttrServiceHost: []tailcfg.RawMessage{tailcfg.RawMessage(svcIPMapJSON)}, + } + nm.SelfNode = self.View() + + matches, err := filter.MatchesFromFilterRules([]tailcfg.FilterRule{ + { + SrcIPs: []string{"100.150.151.152"}, + CapGrant: []tailcfg.CapGrant{{ + Dsts: []netip.Prefix{ + netip.MustParsePrefix("100.101.101.101/32"), + }, + CapMap: tailcfg.PeerCapMap{ + "example.com/cap/interesting": []tailcfg.RawMessage{ + `{"role": "🐿"}`, + }, + }, + }}, + }, + { + SrcIPs: []string{"100.150.151.153"}, + CapGrant: []tailcfg.CapGrant{{ + Dsts: []netip.Prefix{ + netip.MustParsePrefix("100.101.101.101/32"), + }, + CapMap: tailcfg.PeerCapMap{ + "example.com/cap/boring": []tailcfg.RawMessage{ + `{"role": "Viewer"}`, + }, + "example.com/cap/irrelevant": []tailcfg.RawMessage{ + `{"role": "Editor"}`, + }, + }, + }}, + }, + }) + if err != nil { + t.Fatal(err) + } + nm.PacketFilter = matches + b.SetControlClientStatus(nil, controlclient.Status{NetMap: nm}) + + testServ := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + for key, val := range r.Header { + w.Header().Add(key, strings.Join(val, ",")) + } + })) + defer testServ.Close() + + conf := &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:foo": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 443: {HTTPS: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "foo.example.com:443": {Handlers: map[string]*ipn.HTTPHandler{ + "/": { + Proxy: testServ.URL, + AcceptAppCaps: []tailcfg.PeerCapability{"example.com/cap/interesting", "example.com/cap/boring"}, + }, + }}, + }, + }, + }, + } + if err := b.SetServeConfig(conf, ""); err != nil { + t.Fatal(err) + } + + tests := []struct { + name string + srcIP string + wantCap string + }{ + { + name: "request-from-user-within-tailnet", + srcIP: "100.150.151.152", + wantCap: `{"example.com/cap/interesting":[{"role":"🐿"}]}`, + }, + { + name: "request-from-tagged-node-within-tailnet", + srcIP: "100.150.151.153", + wantCap: `{"example.com/cap/boring":[{"role":"Viewer"}]}`, + }, + { + name: "request-from-outside-tailnet", + srcIP: "100.160.161.162", + wantCap: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := &http.Request{ + Host: "foo.example.com", + URL: &url.URL{Path: "/"}, + TLS: &tls.ConnectionState{ServerName: "foo.example.com"}, + } + req = req.WithContext(serveHTTPContextKey.WithValue(req.Context(), &serveHTTPContext{ + ForVIPService: "svc:foo", + DestPort: 443, + SrcAddr: netip.MustParseAddrPort(tt.srcIP + ":1234"), + })) + + w := httptest.NewRecorder() + b.serveWebHandler(w, req) + + dec := new(mime.WordDecoder) + maybeEncoded := w.Result().Header.Get("Tailscale-App-Capabilities") + got, err := dec.DecodeHeader(maybeEncoded) + if err != nil { + t.Fatalf("invalid %q header; failed to decode: %v", maybeEncoded, err) + } + if got != tt.wantCap { + t.Errorf("invalid %q header; want=%q, got=%q", "Tailscale-App-Capabilities", tt.wantCap, got) + } + }) + } +} + func Test_reverseProxyConfiguration(t *testing.T) { b := newTestBackend(t) type test struct { From 7212cd68cc88525318ff62e2a844c573fe871d99 Mon Sep 17 00:00:00 2001 From: Ryan Mulligan Date: Thu, 26 Mar 2026 19:48:27 -0700 Subject: [PATCH 07/15] ipn/ipnlocal: use node peer caps for ingress --- ipn/ipnlocal/node_backend.go | 28 ---------------------------- ipn/ipnlocal/serve.go | 7 +------ ipn/ipnlocal/serve_test.go | 17 ++--------------- 3 files changed, 3 insertions(+), 49 deletions(-) diff --git a/ipn/ipnlocal/node_backend.go b/ipn/ipnlocal/node_backend.go index e026cfa02774e..75550b3d5d5f7 100644 --- a/ipn/ipnlocal/node_backend.go +++ b/ipn/ipnlocal/node_backend.go @@ -295,14 +295,6 @@ func (nb *nodeBackend) PeerCaps(src netip.Addr) tailcfg.PeerCapMap { return nb.peerCapsLocked(src) } -// peerCapsForService returns the capabilities that remote src IP has to the -// specified VIP service hosted by this node. -func (nb *nodeBackend) peerCapsForService(src netip.Addr, serviceName tailcfg.ServiceName) tailcfg.PeerCapMap { - nb.mu.Lock() - defer nb.mu.Unlock() - return nb.peerCapsForServiceLocked(src, serviceName) -} - func (nb *nodeBackend) peerCapsLocked(src netip.Addr) tailcfg.PeerCapMap { if nb.netMap == nil { return nil @@ -325,26 +317,6 @@ func (nb *nodeBackend) peerCapsLocked(src netip.Addr) tailcfg.PeerCapMap { return nil } -func (nb *nodeBackend) peerCapsForServiceLocked(src netip.Addr, serviceName tailcfg.ServiceName) tailcfg.PeerCapMap { - if nb.netMap == nil || serviceName == "" { - return nil - } - filt := nb.filterAtomic.Load() - if filt == nil { - return nil - } - serviceIPMap := nb.netMap.GetVIPServiceIPMap() - if len(serviceIPMap) == 0 { - return nil - } - for _, dst := range serviceIPMap[serviceName] { - if dst.BitLen() == src.BitLen() { // match on family - return filt.CapsWithValues(src, dst) - } - } - return nil -} - // PeerHasCap reports whether the peer contains the given capability string, // with any value(s). func (nb *nodeBackend) PeerHasCap(peer tailcfg.NodeView, wantCap tailcfg.PeerCapability) bool { diff --git a/ipn/ipnlocal/serve.go b/ipn/ipnlocal/serve.go index afd10ab2a321a..1281ecc161c35 100644 --- a/ipn/ipnlocal/serve.go +++ b/ipn/ipnlocal/serve.go @@ -1101,12 +1101,7 @@ func (b *LocalBackend) addAppCapabilitiesHeader(r *httputil.ProxyRequest) error if acceptCaps.IsNil() { return nil } - var peerCaps tailcfg.PeerCapMap - if c.ForVIPService != "" { - peerCaps = b.currentNode().peerCapsForService(c.SrcAddr.Addr(), c.ForVIPService) - } else { - peerCaps = b.PeerCaps(c.SrcAddr.Addr()) - } + peerCaps := b.PeerCaps(c.SrcAddr.Addr()) if peerCaps == nil { return nil } diff --git a/ipn/ipnlocal/serve_test.go b/ipn/ipnlocal/serve_test.go index 72888245c8e1f..00d4b153b7f55 100644 --- a/ipn/ipnlocal/serve_test.go +++ b/ipn/ipnlocal/serve_test.go @@ -1012,26 +1012,13 @@ func TestServeHTTPProxyGrantHeader(t *testing.T) { func TestServeHTTPProxyGrantHeaderForVIPService(t *testing.T) { b := newTestBackend(t) - svcIPMapJSON, err := json.Marshal(tailcfg.ServiceIPMappings{ - "svc:foo": {netip.MustParseAddr("100.101.101.101")}, - }) - if err != nil { - t.Fatal(err) - } - nm := b.NetMap() - self := nm.SelfNode.AsStruct() - self.CapMap = tailcfg.NodeCapMap{ - tailcfg.NodeAttrServiceHost: []tailcfg.RawMessage{tailcfg.RawMessage(svcIPMapJSON)}, - } - nm.SelfNode = self.View() - matches, err := filter.MatchesFromFilterRules([]tailcfg.FilterRule{ { SrcIPs: []string{"100.150.151.152"}, CapGrant: []tailcfg.CapGrant{{ Dsts: []netip.Prefix{ - netip.MustParsePrefix("100.101.101.101/32"), + netip.MustParsePrefix("100.150.151.151/32"), }, CapMap: tailcfg.PeerCapMap{ "example.com/cap/interesting": []tailcfg.RawMessage{ @@ -1044,7 +1031,7 @@ func TestServeHTTPProxyGrantHeaderForVIPService(t *testing.T) { SrcIPs: []string{"100.150.151.153"}, CapGrant: []tailcfg.CapGrant{{ Dsts: []netip.Prefix{ - netip.MustParsePrefix("100.101.101.101/32"), + netip.MustParsePrefix("100.150.151.151/32"), }, CapMap: tailcfg.PeerCapMap{ "example.com/cap/boring": []tailcfg.RawMessage{ From 5aade97528df123536a8e5dcfe18d766509938fd Mon Sep 17 00:00:00 2001 From: Victor Fuentes Date: Fri, 27 Mar 2026 00:39:06 -0700 Subject: [PATCH 08/15] ipn/ipnlocal: add tagged serve identity headers --- ipn/ipnlocal/serve.go | 7 +++++-- ipn/ipnlocal/serve_test.go | 12 ++++++++---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/ipn/ipnlocal/serve.go b/ipn/ipnlocal/serve.go index 1281ecc161c35..d3a60f7ec47ef 100644 --- a/ipn/ipnlocal/serve.go +++ b/ipn/ipnlocal/serve.go @@ -1048,6 +1048,7 @@ func (b *LocalBackend) addTailscaleIdentityHeaders(r *httputil.ProxyRequest) { r.Out.Header.Del("Tailscale-User-Login") r.Out.Header.Del("Tailscale-User-Name") r.Out.Header.Del("Tailscale-User-Profile-Pic") + r.Out.Header.Del("Tailscale-Caller-Tags") r.Out.Header.Del("Tailscale-Funnel-Request") r.Out.Header.Del("Tailscale-Headers-Info") @@ -1064,8 +1065,10 @@ func (b *LocalBackend) addTailscaleIdentityHeaders(r *httputil.ProxyRequest) { return // traffic from outside of Tailnet (funneled or local machine) } if node.IsTagged() { - // 2023-06-14: Not setting identity headers for tagged nodes. - // Only currently set for nodes with user identities. + tags := strings.Join(node.Tags().AsSlice(), ",") + r.Out.Header.Set("Tailscale-User-Login", encTailscaleHeaderValue(tags)) + r.Out.Header.Set("Tailscale-Caller-Tags", encTailscaleHeaderValue(tags)) + r.Out.Header.Set("Tailscale-Headers-Info", "https://tailscale.com/s/serve-headers") return } r.Out.Header.Set("Tailscale-User-Login", encTailscaleHeaderValue(user.LoginName)) diff --git a/ipn/ipnlocal/serve_test.go b/ipn/ipnlocal/serve_test.go index 00d4b153b7f55..755a37bfa65d4 100644 --- a/ipn/ipnlocal/serve_test.go +++ b/ipn/ipnlocal/serve_test.go @@ -814,10 +814,11 @@ func TestServeHTTPProxyHeaders(t *testing.T) { wantHeaders: []headerCheck{ {"X-Forwarded-Proto", "https"}, {"X-Forwarded-For", "100.150.151.153"}, - {"Tailscale-User-Login", ""}, + {"Tailscale-User-Login", "tag:server,tag:test"}, {"Tailscale-User-Name", ""}, {"Tailscale-User-Profile-Pic", ""}, - {"Tailscale-Headers-Info", ""}, + {"Tailscale-Caller-Tags", "tag:server,tag:test"}, + {"Tailscale-Headers-Info", "https://tailscale.com/s/serve-headers"}, }, }, { @@ -829,6 +830,7 @@ func TestServeHTTPProxyHeaders(t *testing.T) { {"Tailscale-User-Login", ""}, {"Tailscale-User-Name", ""}, {"Tailscale-User-Profile-Pic", ""}, + {"Tailscale-Caller-Tags", ""}, {"Tailscale-Headers-Info", ""}, }, }, @@ -955,10 +957,11 @@ func TestServeHTTPProxyGrantHeader(t *testing.T) { wantHeaders: []headerCheck{ {"X-Forwarded-Proto", "https"}, {"X-Forwarded-For", "100.150.151.153"}, - {"Tailscale-User-Login", ""}, + {"Tailscale-User-Login", "tag:server,tag:test"}, {"Tailscale-User-Name", ""}, {"Tailscale-User-Profile-Pic", ""}, - {"Tailscale-Headers-Info", ""}, + {"Tailscale-Caller-Tags", "tag:server,tag:test"}, + {"Tailscale-Headers-Info", "https://tailscale.com/s/serve-headers"}, {"Tailscale-App-Capabilities", `{"example.com/cap/boring":[{"role":"Viewer"}]}`}, }, }, @@ -971,6 +974,7 @@ func TestServeHTTPProxyGrantHeader(t *testing.T) { {"Tailscale-User-Login", ""}, {"Tailscale-User-Name", ""}, {"Tailscale-User-Profile-Pic", ""}, + {"Tailscale-Caller-Tags", ""}, {"Tailscale-Headers-Info", ""}, {"Tailscale-App-Capabilities", ""}, }, From de0a44a74eb4853b7f28d9c569018e8a911c07f2 Mon Sep 17 00:00:00 2001 From: Ryan Mulligan Date: Fri, 27 Mar 2026 08:23:37 -0700 Subject: [PATCH 09/15] kube,ipn/store: fix HA ingress custom TLS cert handling --- ipn/store/kubestore/store_kube.go | 41 ++++++++------- ipn/store/kubestore/store_kube_test.go | 69 ++++++++++++++++++++++---- kube/certs/certs.go | 34 +++++++++++++ kube/certs/certs_test.go | 14 ++++++ 4 files changed, 129 insertions(+), 29 deletions(-) diff --git a/ipn/store/kubestore/store_kube.go b/ipn/store/kubestore/store_kube.go index f7d1b90cd1e2c..fd1c17e3c8c8e 100644 --- a/ipn/store/kubestore/store_kube.go +++ b/ipn/store/kubestore/store_kube.go @@ -215,33 +215,22 @@ func (s *Store) ReadTLSCertAndKey(domain string) (cert, key []byte, err error) { secret, err := s.client.GetSecret(ctx, domain) if err != nil { if kubeclient.IsNotFoundErr(err) { - // TODO(irbekrm): we should return a more specific error - // that wraps ipn.ErrStateNotExist here. - return nil, nil, ipn.ErrStateNotExist + return s.readTLSCertAndKeyFromStateSecret(ctx, certKey, keyKey) } st, ok := err.(*kubeapi.Status) if ok && st.Code == http.StatusForbidden && (s.certShareMode == "ro" || s.certShareMode == "rw") { - // In cert share mode, we read from a dedicated Secret per domain. - // To get here, we already had a cache miss from our in-memory - // store. For write replicas, that means it wasn't available on - // start and it wasn't written since. For read replicas, that means - // it wasn't available on start and it hasn't been reloaded in the - // background. So getting a "forbidden" error is an expected - // "not found" case where we've been asked for a cert we don't - // expect to issue, and so the forbidden error reflects that the - // operator didn't assign permission for a Secret for that domain. - // - // This code path gets triggered by the admin UI's machine page, - // which queries for the node's own TLS cert existing via the - // "tls-cert-status" c2n API. - return nil, nil, ipn.ErrStateNotExist + // In cert share mode, we normally read from a dedicated Secret per + // domain. However, externally managed custom TLS certs for HA + // ingress proxies may exist only in the pod's state Secret. Fall + // back to the state Secret before treating this as a cache miss. + return s.readTLSCertAndKeyFromStateSecret(ctx, certKey, keyKey) } return nil, nil, fmt.Errorf("getting TLS Secret %q: %w", domain, err) } cert = secret.Data[keyTLSCert] key = secret.Data[keyTLSKey] if len(cert) == 0 || len(key) == 0 { - return nil, nil, ipn.ErrStateNotExist + return s.readTLSCertAndKeyFromStateSecret(ctx, certKey, keyKey) } // TODO(irbekrm): a read between these two separate writes would // get a mismatched cert and key. Allow writing both cert and @@ -260,6 +249,22 @@ func (s *Store) ReadTLSCertAndKey(domain string) (cert, key []byte, err error) { return cert, key, nil } +func (s *Store) readTLSCertAndKeyFromStateSecret(ctx context.Context, certKey, keyKey string) ([]byte, []byte, error) { + stateSecret, err := s.client.GetSecret(ctx, s.secretName) + if err != nil { + if kubeclient.IsNotFoundErr(err) { + return nil, nil, ipn.ErrStateNotExist + } + return nil, nil, fmt.Errorf("getting TLS state Secret %q: %w", s.secretName, err) + } + cert := stateSecret.Data[sanitizeKey(certKey)] + key := stateSecret.Data[sanitizeKey(keyKey)] + if len(cert) == 0 || len(key) == 0 { + return nil, nil, ipn.ErrStateNotExist + } + return cert, key, nil +} + func (s *Store) updateSecret(data map[string][]byte, secretName string) (err error) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer func() { diff --git a/ipn/store/kubestore/store_kube_test.go b/ipn/store/kubestore/store_kube_test.go index 1e6f711d686e2..c1b6e6440c1ae 100644 --- a/ipn/store/kubestore/store_kube_test.go +++ b/ipn/store/kubestore/store_kube_test.go @@ -431,15 +431,17 @@ func TestReadTLSCertAndKey(t *testing.T) { ) tests := []struct { - name string - memoryStore map[ipn.StateKey][]byte // pre-existing memory store state - certShareMode string - domain string - secretData map[string][]byte // data to return from mock GetSecret - secretGetErr error // error to return from mock GetSecret - wantCert []byte - wantKey []byte - wantErr error + name string + memoryStore map[ipn.StateKey][]byte // pre-existing memory store state + certShareMode string + domain string + secretData map[string][]byte // data to return from mock GetSecret + secretGetErr error // error to return from mock GetSecret + secretDataByName map[string]map[string][]byte + secretGetErrByName map[string]error + wantCert []byte + wantKey []byte + wantErr error // what should end up in memory store after the store is created wantMemoryStore map[ipn.StateKey][]byte }{ @@ -488,6 +490,38 @@ func TestReadTLSCertAndKey(t *testing.T) { wantCert: []byte(testCert), wantKey: []byte(testKey), }, + { + name: "cert_share_ro_mode_fallback_to_state_secret", + certShareMode: "ro", + domain: testDomain, + secretDataByName: map[string]map[string][]byte{ + "ts-state": { + testDomain + ".crt": []byte(testCert), + testDomain + ".key": []byte(testKey), + }, + }, + secretGetErrByName: map[string]error{ + testDomain: &kubeapi.Status{Code: 404}, + }, + wantCert: []byte(testCert), + wantKey: []byte(testKey), + }, + { + name: "cert_share_rw_mode_fallback_to_state_secret", + certShareMode: "rw", + domain: testDomain, + secretDataByName: map[string]map[string][]byte{ + "ts-state": { + testDomain + ".crt": []byte(testCert), + testDomain + ".key": []byte(testKey), + }, + }, + secretGetErrByName: map[string]error{ + testDomain: &kubeapi.Status{Code: 404}, + }, + wantCert: []byte(testCert), + wantKey: []byte(testKey), + }, { name: "cert_share_ro_mode_found_in_memory", certShareMode: "ro", @@ -514,8 +548,11 @@ func TestReadTLSCertAndKey(t *testing.T) { name: "cert_share_ro_mode_forbidden", certShareMode: "ro", domain: testDomain, - secretGetErr: &kubeapi.Status{Code: 403}, - wantErr: ipn.ErrStateNotExist, + secretGetErrByName: map[string]error{ + testDomain: &kubeapi.Status{Code: 403}, + "ts-state": &kubeapi.Status{Code: 404}, + }, + wantErr: ipn.ErrStateNotExist, }, { name: "cert_share_ro_mode_empty_cert_in_secret", @@ -541,6 +578,16 @@ func TestReadTLSCertAndKey(t *testing.T) { client := &kubeclient.FakeClient{ GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) { + if tt.secretGetErrByName != nil { + if err, ok := tt.secretGetErrByName[name]; ok { + return nil, err + } + } + if tt.secretDataByName != nil { + if data, ok := tt.secretDataByName[name]; ok { + return &kubeapi.Secret{Data: data}, nil + } + } if tt.secretGetErr != nil { return nil, tt.secretGetErr } diff --git a/kube/certs/certs.go b/kube/certs/certs.go index 4c8ac88b6b624..297484f170f75 100644 --- a/kube/certs/certs.go +++ b/kube/certs/certs.go @@ -71,6 +71,17 @@ func (cm *CertManager) EnsureCertLoops(ctx context.Context, sc *ipn.ServeConfig) } } } + if len(currentDomains) > 0 { + certDomains, err := cm.certDomains(ctx) + if err != nil { + return fmt.Errorf("error getting cert domains: %w", err) + } + for domain := range currentDomains { + if !certDomains[domain] { + delete(currentDomains, domain) + } + } + } cm.mu.Lock() defer cm.mu.Unlock() for domain := range currentDomains { @@ -94,6 +105,29 @@ func (cm *CertManager) EnsureCertLoops(ctx context.Context, sc *ipn.ServeConfig) return nil } +func (cm *CertManager) certDomains(ctx context.Context) (map[string]bool, error) { + w, err := cm.lc.WatchIPNBus(ctx, ipn.NotifyInitialNetMap) + if err != nil { + return nil, fmt.Errorf("error watching IPN bus: %w", err) + } + defer w.Close() + + for { + n, err := w.Next() + if err != nil { + return nil, err + } + if n.NetMap == nil { + continue + } + certDomains := make(map[string]bool, len(n.NetMap.DNS.CertDomains)) + for _, domain := range n.NetMap.DNS.CertDomains { + certDomains[domain] = true + } + return certDomains, nil + } +} + // runCertLoop: // - calls localAPI certificate endpoint to ensure that certs are issued for the // given domain name diff --git a/kube/certs/certs_test.go b/kube/certs/certs_test.go index f3662f6c39ad4..3bdd85430f15d 100644 --- a/kube/certs/certs_test.go +++ b/kube/certs/certs_test.go @@ -99,6 +99,20 @@ func TestEnsureCertLoops(t *testing.T) { }, initialGoroutines: 1, // only one loop for the 443 endpoint }, + { + name: "ignore_custom_tls_domains", + initialConfig: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:my-app": { + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "my-app.tailnetxyz.ts.net:443": {}, + "my-app.example.com:443": {}, + }, + }, + }, + }, + initialGoroutines: 1, + }, { name: "remove_domain", initialConfig: &ipn.ServeConfig{ From cba48a51ee00138fc347c8c4ff8f369ffbaa6cbd Mon Sep 17 00:00:00 2001 From: Victor Fuentes Date: Mon, 6 Apr 2026 16:11:05 -0700 Subject: [PATCH 10/15] cmd/k8s-operator: add tailscale.com/service-name annotation for Ingress MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When multiple environments share the same custom domain structure (e.g., zerg.staging.zergrush.dev, zerg.testing.zergrush.dev), the hostnameForIngress function derives the same service name (svc:zerg) for all of them because it only reads the first DNS label of the TLS host. Add a tailscale.com/service-name annotation that takes precedence over the TLS host for service name derivation. This lets operators explicitly control the Tailscale Service name per Ingress: annotations: tailscale.com/proxy-group: zerg-west1-staging-ingress tailscale.com/service-name: zerg-west1-staging The annotation is optional — without it, existing behavior is unchanged (service name derived from first DNS label of TLS host). --- cmd/k8s-operator/ingress.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/cmd/k8s-operator/ingress.go b/cmd/k8s-operator/ingress.go index 67ae14c5822b5..3c3746357088c 100644 --- a/cmd/k8s-operator/ingress.go +++ b/cmd/k8s-operator/ingress.go @@ -442,9 +442,17 @@ func isHTTPRedirectEnabled(ing *networkingv1.Ingress) bool { } // hostnameForIngress returns the hostname for an Ingress resource. -// If the Ingress has TLS configured with a host, it returns the first component of that host. -// Otherwise, it returns a hostname derived from the Ingress name and namespace. +// This hostname is used as the Tailscale Service name (svc:) +// and the first label of the MagicDNS name. +// +// Priority: +// 1. tailscale.com/service-name annotation (explicit override) +// 2. First DNS label of the first TLS host +// 3. Fallback: --ingress func hostnameForIngress(ing *networkingv1.Ingress) string { + if name, ok := ing.Annotations["tailscale.com/service-name"]; ok && name != "" { + return name + } if ing.Spec.TLS != nil && len(ing.Spec.TLS) > 0 && len(ing.Spec.TLS[0].Hosts) > 0 { h := ing.Spec.TLS[0].Hosts[0] hostname, _, _ := strings.Cut(h, ".") From d31e4ac7f24cdb82902162e981ef22b4a92804ef Mon Sep 17 00:00:00 2001 From: Victor Fuentes Date: Tue, 7 Apr 2026 00:39:01 -0700 Subject: [PATCH 11/15] cmd/k8s-operator: allow multiple hosts in single TLS entry Relax the Ingress TLS validation to allow multiple hosts within one TLS entry. The first host is used for service name derivation (or the tailscale.com/service-name annotation). Additional hosts are served using the same custom TLS cert from secretName. This is needed for serving both the primary domain and apex domain (e.g., zerg.zergrush.dev + zergrush.dev) with the same wildcard cert. Only one TLS entry is still enforced (multiple entries are rejected). --- cmd/k8s-operator/ingress-for-pg.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cmd/k8s-operator/ingress-for-pg.go b/cmd/k8s-operator/ingress-for-pg.go index f36b0259f1091..d28df6e081b62 100644 --- a/cmd/k8s-operator/ingress-for-pg.go +++ b/cmd/k8s-operator/ingress-for-pg.go @@ -670,9 +670,11 @@ func (r *HAIngressReconciler) validateIngress(ctx context.Context, ing *networki } } - // Validate TLS configuration - if len(ing.Spec.TLS) > 0 && (len(ing.Spec.TLS) > 1 || len(ing.Spec.TLS[0].Hosts) > 1) { - errs = append(errs, fmt.Errorf("Ingress contains invalid TLS block %v: only a single TLS entry with a single host is allowed", ing.Spec.TLS)) + // Validate TLS configuration — allow multiple hosts in a single TLS entry + // (additional hosts beyond the first are served using the same custom cert). + // Only one TLS entry is allowed. + if len(ing.Spec.TLS) > 1 { + errs = append(errs, fmt.Errorf("Ingress contains invalid TLS block %v: only a single TLS entry is allowed (multiple hosts within one entry are OK)", ing.Spec.TLS)) } // Validate that the hostname will be a valid DNS label From e1c7fe4ad63d0a5cc7ca8defb9f1992516badcb3 Mon Sep 17 00:00:00 2001 From: Victor Fuentes Date: Tue, 7 Apr 2026 10:37:15 -0700 Subject: [PATCH 12/15] cmd/k8s-operator: serve custom TLS certs for all hosts in multi-host TLS entry Previously, when an Ingress had multiple hosts in a single TLS entry (e.g. hosts: ["zerg.zergrush.dev", "zergrush.dev"]), only the first host got the custom cert registered. Additional hosts failed with TLS internal error because the proxy couldn't find a cached cert for them. Fix by changing ingressCustomTLS.host (string) to hosts ([]string) and propagating all hosts through the cert registration pipeline: - copyCustomTLSSecretData writes cert/key data for every host - ingressHTTPSHosts includes all custom hosts in serve config - CustomTLSCerts map is populated for all hosts - handlersForIngress accepts rules matching any TLS host --- cmd/k8s-operator/ingress-for-pg.go | 7 +++-- cmd/k8s-operator/ingress.go | 26 +++++++++-------- cmd/k8s-operator/ingress_tls.go | 45 ++++++++++++++++++++++-------- 3 files changed, 53 insertions(+), 25 deletions(-) diff --git a/cmd/k8s-operator/ingress-for-pg.go b/cmd/k8s-operator/ingress-for-pg.go index d28df6e081b62..3b25b8bd70fb6 100644 --- a/cmd/k8s-operator/ingress-for-pg.go +++ b/cmd/k8s-operator/ingress-for-pg.go @@ -255,8 +255,8 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin return false, fmt.Errorf("error determining DNS name for service: %w", err) } httpsHost := dnsName - if customTLS != nil { - httpsHost = customTLS.host + if customTLS != nil && len(customTLS.hosts) > 0 { + httpsHost = customTLS.hosts[0] } serviceHosts := ingressHTTPSHosts(dnsName, customTLS) @@ -273,7 +273,8 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin logger.Infof("no Ingress serve config ConfigMap found, unable to update serve config. Ensure that ProxyGroup is healthy.") return svcsChanged, nil } - handlers, err := handlersForIngress(ctx, ing, r.Client, r.recorder, httpsHost, logger) + tlsHosts := ingressTLSHosts(ing) + handlers, err := handlersForIngress(ctx, ing, r.Client, r.recorder, tlsHosts, logger) if err != nil { return false, fmt.Errorf("failed to get handlers for Ingress: %w", err) } diff --git a/cmd/k8s-operator/ingress.go b/cmd/k8s-operator/ingress.go index 3c3746357088c..e0db31fba966c 100644 --- a/cmd/k8s-operator/ingress.go +++ b/cmd/k8s-operator/ingress.go @@ -192,11 +192,8 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga } } - var tlsHost string // hostname or FQDN or empty - if ing.Spec.TLS != nil && len(ing.Spec.TLS) > 0 && len(ing.Spec.TLS[0].Hosts) > 0 { - tlsHost = ing.Spec.TLS[0].Hosts[0] - } - handlers, err := handlersForIngress(ctx, ing, a.Client, a.recorder, tlsHost, logger) + tlsHosts := ingressTLSHosts(ing) + handlers, err := handlersForIngress(ctx, ing, a.Client, a.recorder, tlsHosts, logger) if err != nil { return fmt.Errorf("failed to get handlers for ingress: %w", err) } @@ -246,7 +243,10 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga LoginServer: a.ssr.loginServer, } if customTLS != nil { - sts.CustomTLSCerts = map[string]*corev1.Secret{customTLS.host: customTLS.secret} + sts.CustomTLSCerts = make(map[string]*corev1.Secret, len(customTLS.hosts)) + for _, h := range customTLS.hosts { + sts.CustomTLSCerts[h] = customTLS.secret + } } if val := ing.GetAnnotations()[AnnotationExperimentalForwardClusterTrafficViaL7IngresProxy]; val == "true" { @@ -273,8 +273,8 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga } hostname := dev.ingressDNSName - if customTLS != nil { - hostname = customTLS.host + if customTLS != nil && len(customTLS.hosts) > 0 { + hostname = customTLS.hosts[0] } logger.Debugf("setting Ingress hostname to %q", hostname) ports := []networkingv1.IngressPortStatus{} @@ -362,7 +362,7 @@ func parseAcceptAppCaps(ing *networkingv1.Ingress, rec record.EventRecorder) []t return caps } -func handlersForIngress(ctx context.Context, ing *networkingv1.Ingress, cl client.Client, rec record.EventRecorder, tlsHost string, logger *zap.SugaredLogger) (handlers map[string]*ipn.HTTPHandler, err error) { +func handlersForIngress(ctx context.Context, ing *networkingv1.Ingress, cl client.Client, rec record.EventRecorder, tlsHosts []string, logger *zap.SugaredLogger) (handlers map[string]*ipn.HTTPHandler, err error) { acceptAppCaps := parseAcceptAppCaps(ing, rec) addIngressBackend := func(b *networkingv1.IngressBackend, path string) { if path == "" { @@ -412,10 +412,14 @@ func handlersForIngress(ctx context.Context, ing *networkingv1.Ingress, cl clien }) } addIngressBackend(ing.Spec.DefaultBackend, "/") + tlsHostSet := make(map[string]bool, len(tlsHosts)) + for _, h := range tlsHosts { + tlsHostSet[h] = true + } for _, rule := range ing.Spec.Rules { - // Host is optional, but if it's present it must match the TLS host + // Host is optional, but if it's present it must match one of the TLS hosts // otherwise we ignore the rule. - if rule.Host != "" && rule.Host != tlsHost { + if rule.Host != "" && !tlsHostSet[rule.Host] { rec.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "rule with host %q ignored, unsupported", rule.Host) continue } diff --git a/cmd/k8s-operator/ingress_tls.go b/cmd/k8s-operator/ingress_tls.go index 18b423f2babd7..011a356b0b50c 100644 --- a/cmd/k8s-operator/ingress_tls.go +++ b/cmd/k8s-operator/ingress_tls.go @@ -25,14 +25,14 @@ import ( const indexIngressTLSSecret = ".spec.tls.secretName" type ingressCustomTLS struct { - host string + hosts []string secretName string secret *corev1.Secret } func customTLSForIngress(ctx context.Context, cl client.Client, ing *networkingv1.Ingress) (*ingressCustomTLS, error) { - host := ingressTLSHost(ing) - if host == "" || len(ing.Spec.TLS) == 0 || ing.Spec.TLS[0].SecretName == "" { + hosts := ingressTLSHosts(ing) + if len(hosts) == 0 || len(ing.Spec.TLS) == 0 || ing.Spec.TLS[0].SecretName == "" { return nil, nil } @@ -45,15 +45,24 @@ func customTLSForIngress(ctx context.Context, cl client.Client, ing *networkingv } return &ingressCustomTLS{ - host: host, + hosts: hosts, secretName: ing.Spec.TLS[0].SecretName, secret: secret, }, nil } -func ingressTLSHost(ing *networkingv1.Ingress) string { +// ingressTLSHosts returns all hosts from the first TLS entry of the Ingress. +func ingressTLSHosts(ing *networkingv1.Ingress) []string { if ing.Spec.TLS != nil && len(ing.Spec.TLS) > 0 && len(ing.Spec.TLS[0].Hosts) > 0 { - return ing.Spec.TLS[0].Hosts[0] + return ing.Spec.TLS[0].Hosts + } + return nil +} + +// ingressTLSHost returns the first host from the first TLS entry of the Ingress. +func ingressTLSHost(ing *networkingv1.Ingress) string { + if hosts := ingressTLSHosts(ing); len(hosts) > 0 { + return hosts[0] } return "" } @@ -78,9 +87,21 @@ func hasTLSSecretData(ctx context.Context, cl client.Client, ns, name string) (b } func ingressHTTPSHosts(defaultHost string, customTLS *ingressCustomTLS) []string { - hosts := []string{defaultHost} - if customTLS != nil && customTLS.host != defaultHost { - hosts = append([]string{customTLS.host}, hosts...) + if customTLS == nil { + return []string{defaultHost} + } + // Custom TLS hosts come first, followed by the default MagicDNS host + // (if it's not already one of the custom hosts). + seen := make(map[string]bool, len(customTLS.hosts)+1) + var hosts []string + for _, h := range customTLS.hosts { + if !seen[h] { + hosts = append(hosts, h) + seen[h] = true + } + } + if !seen[defaultHost] { + hosts = append(hosts, defaultHost) } return hosts } @@ -89,8 +110,10 @@ func copyCustomTLSSecretData(data map[string][]byte, customTLS *ingressCustomTLS if customTLS == nil { return } - mak.Set(&data, customTLS.host+".crt", append([]byte(nil), customTLS.secret.Data[corev1.TLSCertKey]...)) - mak.Set(&data, customTLS.host+".key", append([]byte(nil), customTLS.secret.Data[corev1.TLSPrivateKeyKey]...)) + for _, host := range customTLS.hosts { + mak.Set(&data, host+".crt", append([]byte(nil), customTLS.secret.Data[corev1.TLSCertKey]...)) + mak.Set(&data, host+".key", append([]byte(nil), customTLS.secret.Data[corev1.TLSPrivateKeyKey]...)) + } } func ensureCustomTLSStateSecrets(ctx context.Context, cl client.Client, namespace string, pg *tsapi.ProxyGroup, customTLS *ingressCustomTLS) error { From 6f0b269bb3619fe22194611b65bf14f196646e0d Mon Sep 17 00:00:00 2001 From: Victor Fuentes Date: Tue, 7 Apr 2026 13:35:03 -0700 Subject: [PATCH 13/15] ipn/ipnlocal: serve custom TLS certs without x509 validation for non-ts.net domains When a user provides a custom TLS cert via Kubernetes Ingress spec.tls, the proxy should serve it regardless of x509 chain or domain validation results. Previously, validCertPEM would reject certs that failed verification (e.g., wildcard cert not covering the apex domain, self-signed cert, private CA), causing TLS internal errors for additional hosts in multi-host TLS entries. Add ReadRaw to certStore that reads cert/key checking only parseability (tls.X509KeyPair), not x509 chain/domain validity. In GetCertPEMWithValidity, when a domain is definitively not ACME-managed (errNotACMEManaged) and doesn't end in .ts.net, fall back to ReadRaw when getCertPEMCached returns errCertExpired. --- ipn/ipnlocal/cert.go | 84 ++++++++++++++++++++++++++++++++++++++- ipn/ipnlocal/cert_test.go | 12 +++--- 2 files changed, 88 insertions(+), 8 deletions(-) diff --git a/ipn/ipnlocal/cert.go b/ipn/ipnlocal/cert.go index a823d798043ee..a5a3c3fb1e230 100644 --- a/ipn/ipnlocal/cert.go +++ b/ipn/ipnlocal/cert.go @@ -133,8 +133,24 @@ func (b *LocalBackend) GetCertPEMWithValidity(ctx context.Context, domain string certDomain, err := b.resolveCertDomain(domain) if err != nil { + // A domain is considered custom (user-provided cert) if: + // 1. It's not in CertDomains (errNotACMEManaged), AND + // 2. It doesn't look like a ts.net domain (to avoid serving + // expired ACME certs for renamed/removed ts.net names). + isCustomDomain := errors.Is(err, errNotACMEManaged) && !strings.HasSuffix(domain, ".ts.net") if pair, cacheErr := getCertPEMCached(cs, domain, now); cacheErr == nil { return pair, nil + } else if isCustomDomain && errors.Is(cacheErr, errCertExpired) { + // The cert exists in the store but failed x509 chain or + // domain validation. For custom domain certs provided by + // the user (e.g., via Kubernetes Ingress spec.tls), serve + // the cert as-is — the user is responsible for ensuring + // the cert is valid for their domain. Only verify that + // the cert/key form a valid TLS keypair. + if pair, rawErr := cs.ReadRaw(domain); rawErr == nil { + return pair, nil + } + return nil, cacheErr } else if cacheErr != nil && !errors.Is(cacheErr, ipn.ErrStateNotExist) { return nil, cacheErr } @@ -292,6 +308,13 @@ type certStore interface { // for now. If they're expired, it returns errCertExpired. // If they don't exist, it returns ipn.ErrStateNotExist. Read(domain string, now time.Time) (*TLSCertKeyPair, error) + // ReadRaw returns the cert and key for domain without performing x509 + // chain or domain validation. It only checks that the cert and key are + // parseable as a valid TLS keypair. This is used for user-provided + // custom TLS certs (non-ts.net domains) where we should serve whatever + // the user provided regardless of CA trust or domain matching. + // If cert/key don't exist, it returns ipn.ErrStateNotExist. + ReadRaw(domain string) (*TLSCertKeyPair, error) // ACMEKey returns the value previously stored via WriteACMEKey. // It is a PEM encoded ECDSA key. ACMEKey() ([]byte, error) @@ -303,6 +326,11 @@ type certStore interface { var errCertExpired = errors.New("cert expired") +// errNotACMEManaged is returned by resolveCertDomain when the requested +// domain is not one of the node's ACME-managed CertDomains. This indicates +// a custom domain (e.g., user-provided TLS cert via Kubernetes Ingress). +var errNotACMEManaged = errors.New("domain is not ACME-managed") + var testX509Roots *x509.CertPool // set non-nil by tests func (b *LocalBackend) getCertStore() (certStore, error) { @@ -386,6 +414,30 @@ func (f certFileStore) Read(domain string, now time.Time) (*TLSCertKeyPair, erro return &TLSCertKeyPair{CertPEM: certPEM, KeyPEM: keyPEM, Cached: true}, nil } +func (f certFileStore) ReadRaw(domain string) (*TLSCertKeyPair, error) { + if !validLookingCertDomain(domain) { + return nil, fmt.Errorf("invalid domain %q", domain) + } + certPEM, err := os.ReadFile(certFile(f.dir, domain)) + if err != nil { + if os.IsNotExist(err) { + return nil, ipn.ErrStateNotExist + } + return nil, err + } + keyPEM, err := os.ReadFile(keyFile(f.dir, domain)) + if err != nil { + if os.IsNotExist(err) { + return nil, ipn.ErrStateNotExist + } + return nil, err + } + if _, err := tls.X509KeyPair(certPEM, keyPEM); err != nil { + return nil, fmt.Errorf("invalid TLS keypair for %q: %w", domain, err) + } + return &TLSCertKeyPair{CertPEM: certPEM, KeyPEM: keyPEM, Cached: true}, nil +} + func (f certFileStore) WriteCert(domain string, cert []byte) error { return atomicfile.WriteFile(certFile(f.dir, domain), cert, 0644) } @@ -447,6 +499,34 @@ func (s certStateStore) Read(domain string, now time.Time) (*TLSCertKeyPair, err return &TLSCertKeyPair{CertPEM: certPEM, KeyPEM: keyPEM, Cached: true}, nil } +func (s certStateStore) ReadRaw(domain string) (*TLSCertKeyPair, error) { + if !validLookingCertDomain(domain) { + return nil, fmt.Errorf("invalid domain %q", domain) + } + if kr, ok := s.StateStore.(TLSCertKeyReader); ok { + cert, key, err := kr.ReadTLSCertAndKey(domain) + if err != nil { + return nil, err + } + if _, err := tls.X509KeyPair(cert, key); err != nil { + return nil, fmt.Errorf("invalid TLS keypair for %q: %w", domain, err) + } + return &TLSCertKeyPair{CertPEM: cert, KeyPEM: key, Cached: true}, nil + } + certPEM, err := s.ReadState(ipn.StateKey(domain + ".crt")) + if err != nil { + return nil, err + } + keyPEM, err := s.ReadState(ipn.StateKey(domain + ".key")) + if err != nil { + return nil, err + } + if _, err := tls.X509KeyPair(certPEM, keyPEM); err != nil { + return nil, fmt.Errorf("invalid TLS keypair for %q: %w", domain, err) + } + return &TLSCertKeyPair{CertPEM: certPEM, KeyPEM: keyPEM, Cached: true}, nil +} + func (s certStateStore) WriteCert(domain string, cert []byte) error { return ipn.WriteState(s.StateStore, ipn.StateKey(domain+".crt"), cert) } @@ -919,7 +999,7 @@ func (b *LocalBackend) resolveCertDomain(domain string) (string, error) { } certDomains := nm.DNS.CertDomains if len(certDomains) == 0 { - return "", errors.New("your Tailscale account does not support getting TLS certs") + return "", fmt.Errorf("your Tailscale account does not support getting TLS certs: %w", errNotACMEManaged) } // Wildcard request like "*.node.ts.net". @@ -938,7 +1018,7 @@ func (b *LocalBackend) resolveCertDomain(domain string) (string, error) { return domain, nil } - return "", fmt.Errorf("invalid domain %q; must be one of %q", domain, certDomains) + return "", fmt.Errorf("invalid domain %q; must be one of %q: %w", domain, certDomains, errNotACMEManaged) } // handleC2NTLSCertStatus returns info about the last TLS certificate issued for the diff --git a/ipn/ipnlocal/cert_test.go b/ipn/ipnlocal/cert_test.go index 06d4f4b9be4e6..cadcdf64730e8 100644 --- a/ipn/ipnlocal/cert_test.go +++ b/ipn/ipnlocal/cert_test.go @@ -118,21 +118,21 @@ func TestResolveCertDomain(t *testing.T) { domain: "app.node.ts.net", certDomains: []string{"node.ts.net"}, hasCap: true, - wantErr: `invalid domain "app.node.ts.net"; must be one of ["node.ts.net"]`, + wantErr: `invalid domain "app.node.ts.net"; must be one of ["node.ts.net"]: domain is not ACME-managed`, }, { name: "subdomain_without_cap_rejected", domain: "app.node.ts.net", certDomains: []string{"node.ts.net"}, hasCap: false, - wantErr: `invalid domain "app.node.ts.net"; must be one of ["node.ts.net"]`, + wantErr: `invalid domain "app.node.ts.net"; must be one of ["node.ts.net"]: domain is not ACME-managed`, }, { name: "multi_level_subdomain_rejected", domain: "a.b.node.ts.net", certDomains: []string{"node.ts.net"}, hasCap: true, - wantErr: `invalid domain "a.b.node.ts.net"; must be one of ["node.ts.net"]`, + wantErr: `invalid domain "a.b.node.ts.net"; must be one of ["node.ts.net"]: domain is not ACME-managed`, }, { name: "wildcard_no_matching_parent", @@ -146,20 +146,20 @@ func TestResolveCertDomain(t *testing.T) { domain: "app.unrelated.ts.net", certDomains: []string{"node.ts.net"}, hasCap: true, - wantErr: `invalid domain "app.unrelated.ts.net"; must be one of ["node.ts.net"]`, + wantErr: `invalid domain "app.unrelated.ts.net"; must be one of ["node.ts.net"]: domain is not ACME-managed`, }, { name: "no_cert_domains", domain: "node.ts.net", certDomains: nil, - wantErr: "your Tailscale account does not support getting TLS certs", + wantErr: "your Tailscale account does not support getting TLS certs: domain is not ACME-managed", }, { name: "wildcard_no_cert_domains", domain: "*.foo.ts.net", certDomains: nil, hasCap: true, - wantErr: "your Tailscale account does not support getting TLS certs", + wantErr: "your Tailscale account does not support getting TLS certs: domain is not ACME-managed", }, { name: "empty_domain", From 12a90553d71423822fdb9e4dee9754467abac422 Mon Sep 17 00:00:00 2001 From: Victor Fuentes Date: Tue, 7 Apr 2026 15:35:43 -0700 Subject: [PATCH 14/15] ipn/store/kubestore: prefer custom TLS certs from state Secret over ACME certs For custom (non-ts.net) domains, check the pod's state Secret before the domain-specific Secret when looking up TLS certs. Custom TLS certs from Ingress spec.tls entries are written to the state Secret by the operator, while ACME-provisioned certs are stored in domain-specific Secrets. Without this ordering, an ACME cert for the same domain would shadow the user-provided custom cert. For ts.net domains, the lookup order is unchanged (domain-specific Secret first) to avoid extra API calls on the common ACME path. --- ipn/store/kubestore/store_kube.go | 39 ++++++++++++++++++++++++-- ipn/store/kubestore/store_kube_test.go | 39 ++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 3 deletions(-) diff --git a/ipn/store/kubestore/store_kube.go b/ipn/store/kubestore/store_kube.go index fd1c17e3c8c8e..8d6777be03506 100644 --- a/ipn/store/kubestore/store_kube.go +++ b/ipn/store/kubestore/store_kube.go @@ -7,6 +7,7 @@ package kubestore import ( "context" "encoding/json" + "errors" "fmt" "net" "net/http" @@ -189,9 +190,18 @@ func (s *Store) WriteTLSCertAndKey(domain string, cert, key []byte) (err error) return nil } -// ReadTLSCertAndKey reads a TLS cert and key from memory or from a -// domain-specific Secret. It first checks the in-memory store, if not found in -// memory and running cert store in read-only mode, looks up a Secret. +// ReadTLSCertAndKey reads a TLS cert and key from memory or from Kubernetes +// Secrets. It first checks the in-memory store, then reads from Kubernetes +// Secrets with ordering that depends on the domain type: +// +// For custom domains (non-ts.net): the pod's state Secret is checked first, +// since custom TLS certs from Ingress spec.tls entries are written there by +// the operator. This ensures user-provided custom certs take precedence over +// any ACME-provisioned certs in domain-specific Secrets. +// +// For ts.net domains: the domain-specific Secret is checked first (standard +// ACME cert sharing path), falling back to the state Secret. +// // Note that write replicas of HA Ingress always retrieve TLS certs from Secrets. func (s *Store) ReadTLSCertAndKey(domain string) (cert, key []byte, err error) { if err := dnsname.ValidHostname(domain); err != nil { @@ -212,6 +222,29 @@ func (s *Store) ReadTLSCertAndKey(domain string) (cert, key []byte, err error) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() + + // For custom (non-ts.net) domains, check the pod's state Secret first. + // Custom TLS certs (from Ingress spec.tls[].secretName) are written + // here by the operator, keyed as "domain.crt" / "domain.key". Checking + // this before the domain-specific Secret ensures custom certs take + // precedence over ACME-provisioned certs for the same domain. + if !strings.HasSuffix(domain, ".ts.net") { + cert, key, stateErr := s.readTLSCertAndKeyFromStateSecret(ctx, certKey, keyKey) + if stateErr == nil { + if s.certShareMode == "ro" { + s.memory.WriteState(ipn.StateKey(certKey), cert) + s.memory.WriteState(ipn.StateKey(keyKey), key) + } + return cert, key, nil + } + if !errors.Is(stateErr, ipn.ErrStateNotExist) { + // Real error reading state Secret (e.g. API failure). + return nil, nil, stateErr + } + // State Secret didn't have the cert; fall through to + // domain-specific Secret. + } + secret, err := s.client.GetSecret(ctx, domain) if err != nil { if kubeclient.IsNotFoundErr(err) { diff --git a/ipn/store/kubestore/store_kube_test.go b/ipn/store/kubestore/store_kube_test.go index c1b6e6440c1ae..1665dc9c49862 100644 --- a/ipn/store/kubestore/store_kube_test.go +++ b/ipn/store/kubestore/store_kube_test.go @@ -571,6 +571,45 @@ func TestReadTLSCertAndKey(t *testing.T) { secretGetErr: fmt.Errorf("api error"), wantErr: fmt.Errorf("getting TLS Secret %q: api error", sanitizeKey(testDomain)), }, + { + // When both the state Secret (custom cert from Ingress TLS) + // and domain-specific Secret (ACME cert) have data for the + // same custom (non-ts.net) domain, the state Secret should + // take precedence. + name: "cert_share_rw_mode_custom_cert_takes_precedence", + certShareMode: "rw", + domain: "app.example.com", + secretDataByName: map[string]map[string][]byte{ + "ts-state": { + "app.example.com.crt": []byte("custom-cert"), + "app.example.com.key": []byte("custom-key"), + }, + "app.example.com": { + "tls.crt": []byte("acme-cert"), + "tls.key": []byte("acme-key"), + }, + }, + wantCert: []byte("custom-cert"), + wantKey: []byte("custom-key"), + }, + { + // Same test for ro mode. + name: "cert_share_ro_mode_custom_cert_takes_precedence", + certShareMode: "ro", + domain: "app.example.com", + secretDataByName: map[string]map[string][]byte{ + "ts-state": { + "app.example.com.crt": []byte("custom-cert"), + "app.example.com.key": []byte("custom-key"), + }, + "app.example.com": { + "tls.crt": []byte("acme-cert"), + "tls.key": []byte("acme-key"), + }, + }, + wantCert: []byte("custom-cert"), + wantKey: []byte("custom-key"), + }, } for _, tt := range tests { From 222c9b58fba09c3f84f40caf944c6a98cfa8b310 Mon Sep 17 00:00:00 2001 From: Victor Fuentes Date: Tue, 7 Apr 2026 18:22:01 -0700 Subject: [PATCH 15/15] ipn/ipnlocal: serve custom TLS certs for ACME-managed non-ts.net domains When a non-ts.net domain is both ACME-managed (in CertDomains) and has a user-provided custom cert in the state store (from Kubernetes Ingress spec.tls), the custom cert may fail x509 chain validation (e.g., different CA, missing intermediates) causing getCertPEMCached to return errCertExpired. Previously this fell through to ACME cert issuance, which could return a wildcard cert that doesn't cover the apex domain. Now, when getCertPEMCached fails for a non-ts.net ACME-managed domain, try ReadRaw to serve the custom cert without chain validation (but still checking it's a valid TLS keypair and not time-expired). This ensures the user-provided cert takes precedence over ACME issuance. Also add a time-expiry check to both ReadRaw fallback paths to avoid serving genuinely expired certs and suppressing ACME renewal. --- ipn/ipnlocal/cert.go | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/ipn/ipnlocal/cert.go b/ipn/ipnlocal/cert.go index a5a3c3fb1e230..c280f90302b71 100644 --- a/ipn/ipnlocal/cert.go +++ b/ipn/ipnlocal/cert.go @@ -144,11 +144,14 @@ func (b *LocalBackend) GetCertPEMWithValidity(ctx context.Context, domain string // The cert exists in the store but failed x509 chain or // domain validation. For custom domain certs provided by // the user (e.g., via Kubernetes Ingress spec.tls), serve - // the cert as-is — the user is responsible for ensuring - // the cert is valid for their domain. Only verify that - // the cert/key form a valid TLS keypair. + // the cert as-is if it's not time-expired — the user is + // responsible for ensuring the cert is valid for their + // domain. Only verify that the cert/key form a valid TLS + // keypair and the cert hasn't expired. if pair, rawErr := cs.ReadRaw(domain); rawErr == nil { - return pair, nil + if crt, parseErr := pair.parseCertificate(); parseErr == nil && now.Before(crt.NotAfter) { + return pair, nil + } } return nil, cacheErr } else if cacheErr != nil && !errors.Is(cacheErr, ipn.ErrStateNotExist) { @@ -165,7 +168,7 @@ func (b *LocalBackend) GetCertPEMWithValidity(ctx context.Context, domain string log.Printf("acme %T: %s", v, j) } - if pair, err := getCertPEMCached(cs, certDomain, now); err == nil { + if pair, cacheErr := getCertPEMCached(cs, certDomain, now); cacheErr == nil { if envknob.IsCertShareReadOnlyMode() { return pair, nil } @@ -194,6 +197,21 @@ func (b *LocalBackend) GetCertPEMWithValidity(ctx context.Context, domain string // If the caller requested a specific validity duration, fall through // to synchronous renewal to fulfill that. logf("starting sync renewal") + } else if !strings.HasSuffix(domain, ".ts.net") && errors.Is(cacheErr, errCertExpired) { + // The cert exists in the store but failed x509 chain or domain + // validation. For non-ts.net domains that are also in CertDomains + // (ACME-managed), still try serving the cert as a user-provided + // custom cert — the user is responsible for ensuring it's valid. + // This covers the case where a domain is both ACME-managed and has + // a custom cert from Ingress spec.tls that fails full x509 + // validation (e.g., different CA, missing intermediates). + // Only serve if the cert is not actually time-expired, to avoid + // suppressing ACME renewal for genuinely expired certs. + if pair, rawErr := cs.ReadRaw(domain); rawErr == nil { + if crt, parseErr := pair.parseCertificate(); parseErr == nil && now.Before(crt.NotAfter) { + return pair, nil + } + } } if envknob.IsCertShareReadOnlyMode() {