e2e: add RFC-18 flex-algo end-to-end tests (RFC-18, PR 4/4)

CHANGELOG.md

@@ -78,6 +78,8 @@ All notable changes to this project will be documented in this file.
   - When flex-algo is enabled: generate IS-IS TE admin-group assignments on link interfaces, flex-algo topology definitions, and `system-colored-tunnel-rib` as the BGP next-hop resolution source
   - Stamp BGP color extcommunity on user tunnel route-maps for tenants with `include_topologies` set
   - Add `node-segment ipv4 index` lines to Vpnv4 loopback config for each flex-algo, sourced from segment routing IDs backfilled by the activator
+- E2E Tests
+  - Add flex-algo e2e tests: topology lifecycle (safe-delete enforcement), topology filter (`--topology` CLI validation), unicast-drained flag transitions, tenant `include_topologies` changes, migration dry-run and live run, and controller config generation against live cEOS
 - Client
   - Add `doublezero_connection_info` Prometheus metric exposing connection metadata (user_type, network, current_device, metro, tunnel_name, tunnel_src, tunnel_dst) ([#3201](https://github.com/malbeclabs/doublezero/pull/3201))
   - Add `doublezero_connection_rtt_nanoseconds` and `doublezero_connection_loss_percentage` Prometheus metrics reporting RTT and packet loss to the current connected device

e2e/docker/controller/entrypoint.sh

@@ -26,4 +26,8 @@ if [ -n "${ALLOY_PROMETHEUS_URL:-}" ]; then
 fi
 
 # Start the controller.
-doublezero-controller start -listen-addr 0.0.0.0 -listen-port 7000 -program-id ${DZ_SERVICEABILITY_PROGRAM_ID} -solana-rpc-endpoint ${DZ_LEDGER_URL} -device-local-asn 65342 -no-hardware
+CONTROLLER_ARGS="-listen-addr 0.0.0.0 -listen-port 7000 -program-id ${DZ_SERVICEABILITY_PROGRAM_ID} -solana-rpc-endpoint ${DZ_LEDGER_URL} -device-local-asn 65342 -no-hardware"
+if [ -n "${DZ_FEATURES_CONFIG_PATH:-}" ]; then
+  CONTROLLER_ARGS="${CONTROLLER_ARGS} -features-config ${DZ_FEATURES_CONFIG_PATH}"
+fi
+doublezero-controller start ${CONTROLLER_ARGS}

e2e/flex_algo_test.go

@@ -0,0 +1,231 @@
+//go:build e2e
+
+package e2e_test
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/malbeclabs/doublezero/e2e/internal/devnet"
+	"github.com/malbeclabs/doublezero/e2e/internal/random"
+	serviceability "github.com/malbeclabs/doublezero/sdk/serviceability/go"
+	"github.com/mr-tron/base58"
+	"github.com/stretchr/testify/require"
+)
+
+// TestE2E_FlexAlgo_BasicWiring verifies the steady-state flex-algo wiring:
+//   - When features.flex_algo.enabled is true, the controller generates
+//     "router traffic-engineering", IS-IS flex-algo, and TE admin-group blocks
+//   - A WAN link tagged with UNICAST-DEFAULT topology gets the admin-group attribute
+//   - Vpnv4 loopbacks with backfilled node segments get the flex-algo node-segment line
+//
+// This test uses normal operational CLI commands (not the one-time migrate tool):
+//   - doublezero link topology create  — to create the topology
+//   - doublezero link topology backfill — to add flex-algo node segments to Loopback255
+//   - doublezero link update --link-topology — to tag the WAN link
+func TestE2E_FlexAlgo_BasicWiring(t *testing.T) {
+	t.Parallel()
+
+	deployID := "dz-e2e-" + t.Name() + "-" + random.ShortID()
+	log := newTestLoggerForTest(t)
+
+	// Write features.yaml enabling flex_algo to a file in the test's temp dir.
+	// FeaturesConfigPath is a host-side path volume-mounted into the controller container.
+	deployDir := t.TempDir()
+	featuresConfigPath := filepath.Join(deployDir, "features.yaml")
+	err := os.WriteFile(featuresConfigPath, []byte("features:\n  flex_algo:\n    enabled: true\n"), 0o644)
+	require.NoError(t, err)
+
+	dn, err := devnet.New(devnet.DevnetSpec{
+		DeployID:  deployID,
+		DeployDir: deployDir,
+
+		CYOANetwork: devnet.CYOANetworkSpec{
+			CIDRPrefix: subnetCIDRPrefix,
+		},
+		Controller: devnet.ControllerSpec{
+			FeaturesConfigPath: featuresConfigPath,
+		},
+	}, log, dockerClient, subnetAllocator)
+	require.NoError(t, err)
+
+	ctx := t.Context()
+
+	log.Debug("==> Starting devnet")
+	err = dn.Start(ctx, nil)
+	require.NoError(t, err)
+	log.Debug("--> Devnet started")
+
+	// Create a Docker network simulating the WAN link between the two devices.
+	linkNetwork := devnet.NewMiscNetwork(dn, log, "la2-dz01:ewr1-dz01")
+	_, err = linkNetwork.CreateIfNotExists(ctx)
+	require.NoError(t, err)
+
+	// Add both devices in parallel. Each device has:
+	//   - Ethernet2: physical WAN interface (on the shared link network)
+	//   - Loopback255: vpnv4 loopback (required for IS-IS SR flex-algo node segments)
+	var wg sync.WaitGroup
+	var device1, device2 *devnet.Device
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		var addErr error
+		device1, addErr = dn.AddDevice(ctx, devnet.DeviceSpec{
+			Code:                         "la2-dz01",
+			Location:                     "lax",
+			Exchange:                     "xlax",
+			CYOANetworkIPHostID:          8,
+			CYOANetworkAllocatablePrefix: 29,
+			AdditionalNetworks:           []string{linkNetwork.Name},
+			Interfaces:                   map[string]string{"Ethernet2": "physical"},
+			LoopbackInterfaces:           map[string]string{"Loopback255": "vpnv4"},
+		})
+		require.NoError(t, addErr)
+		log.Debug("--> Device1 added", "id", device1.ID, "code", device1.Spec.Code)
+	}()
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		var addErr error
+		device2, addErr = dn.AddDevice(ctx, devnet.DeviceSpec{
+			Code:                         "ewr1-dz01",
+			Location:                     "ewr",
+			Exchange:                     "xewr",
+			CYOANetworkIPHostID:          16,
+			CYOANetworkAllocatablePrefix: 29,
+			AdditionalNetworks:           []string{linkNetwork.Name},
+			Interfaces:                   map[string]string{"Ethernet2": "physical"},
+			LoopbackInterfaces:           map[string]string{"Loopback255": "vpnv4"},
+		})
+		require.NoError(t, addErr)
+		log.Debug("--> Device2 added", "id", device2.ID, "code", device2.Spec.Code)
+	}()
+
+	wg.Wait()
+
+	// Create the UNICAST-DEFAULT topology. The smart contract auto-assigns
+	// admin_group_bit and flex_algo_number.
+	log.Debug("==> Creating UNICAST-DEFAULT topology")
+	_, err = dn.Manager.Exec(ctx, []string{
+		"doublezero", "link", "topology", "create",
+		"--name", "unicast-default",
+		"--constraint", "include-any",
+	})
+	require.NoError(t, err)
+	log.Debug("--> UNICAST-DEFAULT topology created")
+
+	// Create a WAN link between the two devices and wait for activation.
+	log.Debug("==> Creating WAN link onchain")
+	_, err = dn.Manager.Exec(ctx, []string{"bash", "-c",
+		`doublezero link create wan \
+			--code "la2-dz01:ewr1-dz01" \
+			--contributor co01 \
+			--side-a la2-dz01 \
+			--side-a-interface Ethernet2 \
+			--side-z ewr1-dz01 \
+			--side-z-interface Ethernet2 \
+			--bandwidth "10 Gbps" \
+			--mtu 2048 \
+			--delay-ms 20 \
+			--jitter-ms 2 \
+			--desired-status activated \
+			-w`,
+	})
+	require.NoError(t, err)
+	log.Debug("--> WAN link created")
+
+	// Wait for the link to be activated onchain and capture its pubkey.
+	log.Debug("==> Waiting for link activation")
+	serviceabilityClient, err := dn.Ledger.GetServiceabilityClient()
+	require.NoError(t, err)
+	var linkPubkey string
+	require.Eventually(t, func() bool {
+		data, err := serviceabilityClient.GetProgramData(ctx)
+		if err != nil {
+			log.Debug("Failed to get program data", "error", err)
+			return false
+		}
+		for _, link := range data.Links {
+			if link.Code == "la2-dz01:ewr1-dz01" {
+				if link.Status == serviceability.LinkStatusActivated {
+					linkPubkey = base58.Encode(link.PubKey[:])
+					return true
+				}
+				log.Debug("Link not yet activated", "status", link.Status)
+				return false
+			}
+		}
+		log.Debug("Link not found yet")
+		return false
+	}, 60*time.Second, 2*time.Second, "link was not activated within timeout")
+	log.Debug("--> Link activated", "pubkey", linkPubkey)
+
+	// Tag the WAN link with the UNICAST-DEFAULT topology.
+	// In steady-state operation this is done when the link is provisioned.
+	log.Debug("==> Tagging WAN link with UNICAST-DEFAULT topology")
+	_, err = dn.Manager.Exec(ctx, []string{
+		"doublezero", "link", "update",
+		"--pubkey", linkPubkey,
+		"--link-topology", "unicast-default",
+	})
+	require.NoError(t, err)
+	log.Debug("--> WAN link tagged")
+
+	// Note: flex-algo node segments are automatically backfilled by the activator when the
+	// topology is created — no manual backfill command is required.
+
+	// Poll the controller-rendered config for each device and assert that all
+	// expected flex-algo sections are present.
+	for _, device := range []*devnet.Device{device1, device2} {
+		device := device
+		t.Run(fmt.Sprintf("controller_config_%s", device.Spec.Code), func(t *testing.T) {
+			log.Debug("==> Checking flex-algo controller config", "device", device.Spec.Code)
+
+			// Expected strings in the EOS config when flex_algo is enabled,
+			// the link is tagged, and node segments are backfilled.
+			want := []string{
+				// router traffic-engineering block
+				"router traffic-engineering",
+				// UNICAST-DEFAULT admin-group alias
+				"administrative-group alias UNICAST-DEFAULT group",
+				// IS-IS flex-algo advertisement
+				"flex-algo UNICAST-DEFAULT level-2 advertised",
+				// Ethernet2 WAN interface TE tagging
+				"traffic-engineering administrative-group UNICAST-DEFAULT",
+				// Loopback255 flex-algo node segment
+				"flex-algo UNICAST-DEFAULT",
+			}
+
+			require.Eventually(t, func() bool {
+				cfg, fetchErr := dn.Controller.GetAgentConfig(ctx, device.ID)
+				if fetchErr != nil {
+					log.Debug("Failed to get agent config", "device", device.Spec.Code, "error", fetchErr)
+					return false
+				}
+				config := cfg.Config
+				for _, s := range want {
+					if !strings.Contains(config, s) {
+						log.Debug("Config missing expected section",
+							"device", device.Spec.Code,
+							"missing", s,
+						)
+						return false
+					}
+				}
+				return true
+			}, 60*time.Second, 2*time.Second,
+				"device %s controller config did not contain flex-algo sections within timeout",
+				device.Spec.Code,
+			)
+
+			log.Debug("--> flex-algo controller config verified", "device", device.Spec.Code)
+		})
+	}
+}

e2e/internal/devnet/cmd/add-device.go

@@ -8,6 +8,18 @@ import (
 	"github.com/spf13/cobra"
 )
 
+// physicalInterfacesForNetworks returns a map of Ethernet interface names to "physical"
+// for each additional network. Ethernet2 maps to the first additional network,
+// Ethernet3 to the second, and so on — matching how cEOS assigns eth interfaces
+// to Docker networks (eth0=Management0, eth1=Ethernet1/CYOA, eth2+=additional).
+func physicalInterfacesForNetworks(count int) map[string]string {
+	interfaces := make(map[string]string, count)
+	for i := range count {
+		interfaces[fmt.Sprintf("Ethernet%d", i+2)] = "physical"
+	}
+	return interfaces
+}
+
 type AddDeviceCmd struct{}
 
 func NewAddDeviceCmd() *AddDeviceCmd {
@@ -52,9 +64,7 @@ func (c *AddDeviceCmd) Command() *cobra.Command {
 					ManagementNS: "ns-management",
 					Verbose:      true,
 				},
-				Interfaces: map[string]string{
-					"Ethernet2": "physical",
-				},
+				Interfaces: physicalInterfacesForNetworks(len(additionalNetworks)),
 				LoopbackInterfaces: map[string]string{
 					"Loopback255": "vpnv4",
 					"Loopback256": "ipv4",

e2e/internal/devnet/cmd/devnet.go

@@ -37,7 +37,7 @@ type LocalDevnet struct {
 	workspaceDir string
 }
 
-func NewLocalDevnet(log *slog.Logger, deployID string) (*LocalDevnet, error) {
+func NewLocalDevnet(log *slog.Logger, deployID string, featuresConfigPath string) (*LocalDevnet, error) {
 	// Set the default logger for testcontainers.
 	logging.SetTestcontainersLogger(log)
 
@@ -90,6 +90,9 @@ func NewLocalDevnet(log *slog.Logger, deployID string) (*LocalDevnet, error) {
 			Verbose:  true,
 			Interval: 10 * time.Second,
 		},
+		Controller: devnet.ControllerSpec{
+			FeaturesConfigPath: featuresConfigPath,
+		},
 	}, log, dockerClient, subnetAllocator)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create devnet: %w", err)

e2e/internal/devnet/cmd/root.go

@@ -42,6 +42,9 @@ func Run() ExitCode {
 	var deployID string
 	rootCmd.PersistentFlags().StringVar(&deployID, "deploy-id", envWithDefault("DZ_DEPLOY_ID", defaultDeployID), "deploy identifier (env: DZ_DEPLOY_ID, default: "+defaultDeployID+")")
 
+	var featuresConfigPath string
+	rootCmd.PersistentFlags().StringVar(&featuresConfigPath, "features-config", "", "path to features.yaml to pass to the controller (optional)")
+
 	rootCmd.AddCommand(
 		NewBuildCmd().Command(),
 		NewStartCmd().Command(),
@@ -77,7 +80,12 @@ func withDevnet(f func(ctx context.Context, dn *LocalDevnet, cmd *cobra.Command,
 			return fmt.Errorf("failed to get deploy-id flag: %w", err)
 		}
 
-		dn, err := NewLocalDevnet(log, deployID)
+		featuresConfigPath, err := cmd.Root().PersistentFlags().GetString("features-config")
+		if err != nil {
+			return fmt.Errorf("failed to get features-config flag: %w", err)
+		}
+
+		dn, err := NewLocalDevnet(log, deployID, featuresConfigPath)
 		if err != nil {
 			return fmt.Errorf("failed to create devnet: %w", err)
 		}

e2e/internal/devnet/controller.go

@@ -26,6 +26,10 @@ const (
 
 type ControllerSpec struct {
 	ContainerImage string
+	// FeaturesConfigPath is the host path to a features.yaml file to mount into the
+	// controller container and pass via -features-config. If empty, the controller
+	// starts without a features config (flex_algo.enabled defaults to false).
+	FeaturesConfigPath string
 }
 
 func (s *ControllerSpec) Validate(cyoaNetworkSpec CYOANetworkSpec) error {
@@ -117,6 +121,18 @@ func (c *Controller) Start(ctx context.Context) error {
 		env["ALLOY_PROMETHEUS_URL"] = c.dn.Prometheus.InternalRemoteWriteURL()
 	}
 
+	const containerFeaturesConfigPath = "/features.yaml"
+	var files []testcontainers.ContainerFile
+	if c.dn.Spec.Controller.FeaturesConfigPath != "" {
+		env["DZ_FEATURES_CONFIG_PATH"] = containerFeaturesConfigPath
+		files = append(files, testcontainers.ContainerFile{
+			HostFilePath:      c.dn.Spec.Controller.FeaturesConfigPath,
+			ContainerFilePath: containerFeaturesConfigPath,
+			FileMode:          0o644,
+		})
+		c.log.Debug("==> Controller features config", "hostPath", c.dn.Spec.Controller.FeaturesConfigPath)
+	}
+
 	req := testcontainers.ContainerRequest{
 		Image: c.dn.Spec.Controller.ContainerImage,
 		Name:  c.dockerContainerName(),
@@ -126,6 +142,7 @@ func (c *Controller) Start(ctx context.Context) error {
 		ExposedPorts: []string{fmt.Sprintf("%d/tcp", internalControllerPort)},
 		WaitingFor:   wait.ForExposedPort(),
 		Env:          env,
+		Files:        files,
 		Networks:     []string{c.dn.DefaultNetwork.Name},
 		NetworkAliases: map[string][]string{
 			c.dn.DefaultNetwork.Name: {"controller"},

e2e/internal/devnet/smartcontract_geolocation.go

@@ -81,7 +81,7 @@ func (dn *Devnet) InitGeolocationProgramConfigIfNotInitialized(ctx context.Conte
 	}, docker.NoPrintOnError())
 	if err != nil {
 		outputStr := strings.ToLower(string(output))
-		if strings.Contains(outputStr, "already") || strings.Contains(outputStr, "already in use") {
+		if strings.Contains(outputStr, "already") || strings.Contains(outputStr, "uninitialized account") {
 			dn.log.Debug("--> Geolocation program config is already initialized")
 			return false, nil
 		}

e2e/link_onchain_allocation_test.go

@@ -55,6 +55,16 @@ func TestE2E_Link_OnchainAllocation(t *testing.T) {
 	err = dn.Start(ctx, nil)
 	require.NoError(t, err)
 
+	// Create UNICAST-DEFAULT topology before any link activation — required by the
+	// onchain program to auto-tag links at activation time.
+	log.Debug("==> Creating UNICAST-DEFAULT topology")
+	_, err = dn.Manager.Exec(ctx, []string{
+		"doublezero", "link", "topology", "create",
+		"--name", "unicast-default",
+		"--constraint", "include-any",
+	})
+	require.NoError(t, err)
+
 	// Create two devices for the link endpoints
 	// Note: Must use globally routable IPs - smart contract rejects private, documentation, and other reserved IPs
 	log.Debug("==> Creating devices for link test")