Expand Visualizer Tests and Patch Visualizer Bugs

docs/source/features/visualization.rst

@@ -109,12 +109,12 @@ You can also configure custom visualizers in the code by defining ``VisualizerCf
                 dock_position="SAME",
                 window_width=1280,
                 window_height=720,
-                camera_position=(0.0, 0.0, 20.0), # high top down view
-                camera_target=(0.0, 0.0, 0.0),
+                eye=(0.0, 0.0, 20.0), # high top down view
+                lookat=(0.0, 0.0, 0.0),
             ),
             NewtonVisualizerCfg(
-                camera_position=(5.0, 5.0, 5.0), # closer quarter view
-                camera_target=(0.0, 0.0, 0.0),
+                eye=(5.0, 5.0, 5.0), # closer quarter view
+                lookat=(0.0, 0.0, 0.0),
                 show_joints=True,
             ),
             RerunVisualizerCfg(
@@ -201,8 +201,8 @@ Omniverse Visualizer
         window_height=720,                        # Viewport height in pixels
 
         # Camera settings
-        camera_position=(8.0, 8.0, 3.0),         # Initial camera position (x, y, z)
-        camera_target=(0.0, 0.0, 0.0),           # Camera look-at target
+        eye=(8.0, 8.0, 3.0),                     # Initial camera position (x, y, z)
+        lookat=(0.0, 0.0, 0.0),                  # Camera look-at target
 
         # Feature toggles
         enable_markers=True,                      # Enable visualization markers
@@ -255,8 +255,8 @@ Newton Visualizer
         window_height=1080,                       # Window height in pixels
 
         # Camera settings
-        camera_position=(8.0, 8.0, 3.0),         # Initial camera position (x, y, z)
-        camera_target=(0.0, 0.0, 0.0),           # Camera look-at target
+        eye=(8.0, 8.0, 3.0),                     # Initial camera position (x, y, z)
+        lookat=(0.0, 0.0, 0.0),                  # Camera look-at target
 
         # Performance tuning
         update_frequency=1,                       # Update every N frames (1=every frame)
@@ -303,8 +303,8 @@ Rerun Visualizer
         bind_address="0.0.0.0",                  # Endpoint host formatting/reuse checks
 
         # Camera settings
-        camera_position=(8.0, 8.0, 3.0),         # Initial camera position (x, y, z)
-        camera_target=(0.0, 0.0, 0.0),           # Camera look-at target
+        eye=(8.0, 8.0, 3.0),                     # Initial camera position (x, y, z)
+        lookat=(0.0, 0.0, 0.0),                  # Camera look-at target
 
         # History settings
         keep_historical_data=False,               # Keep transforms for time scrubbing

source/isaaclab/isaaclab/app/app_launcher.py

@@ -188,6 +188,8 @@ def __init__(self, launcher_args: argparse.Namespace | dict | None = None, **kwa
         self._offscreen_render: bool  # 0: Disabled, 1: Enabled
         self._sim_experience_file: str  # Experience file to load
         self._visualizer_max_worlds: int | None  # Optional max worlds override for Newton-based visualizers
+        self._video_enabled: bool  # Whether --video recording is enabled
+        self._video_auto_start_kit: bool  # Whether headless video should auto-inject Kit visualizer
 
         # Exposed to train scripts
         self.device_id: int  # device ID for GPU simulation (defaults to 0)
@@ -842,14 +844,28 @@ def _resolve_xr_settings(self, launcher_args: dict):
 
     def _resolve_viewport_settings(self, launcher_args: dict):
         """Resolve viewport related settings."""
+        self._video_enabled = bool(launcher_args.get("video", False))
         # Check if we can disable the viewport to improve performance
         #   This should only happen if we are running headless and do not require livestreaming or video recording
         #   This is different from offscreen_render because this only affects the default viewport and
         #   not other render-products in the scene
         self._render_viewport = True
-        if self._headless and not self._livestream and not launcher_args.get("video", False):
+        if self._headless and not self._livestream and not self._video_enabled:
             self._render_viewport = False
 
+        # Auto-start a headless Kit visualizer when recording video without an explicit
+        # visualizer selection. This ensures app.update() is pumped and camera updates
+        # can be forwarded from viewer settings.
+        has_explicit_kit = self._cli_visualizer_explicit and "kit" in set(self._cli_visualizer_types)
+        self._video_auto_start_kit = (
+            self._video_enabled
+            and self._headless
+            and not self._livestream
+            and not self._xr
+            and not self._cli_visualizer_disable_all
+            and not has_explicit_kit
+        )
+
         # hide_ui flag
         launcher_args["hide_ui"] = False
         if self._headless and not self._livestream:
@@ -1069,6 +1085,9 @@ def _load_extensions(self):
         # (no Kit GUI) the AR profile must be enabled programmatically so that
         # the OpenXR session starts without user interaction
         settings.set_bool("/isaaclab/xr/auto_start", self._headless and self._xr)
+        # set setting to indicate video recording mode and optional Kit auto-start
+        settings.set_bool("/isaaclab/video/enabled", self._video_enabled)
+        settings.set_bool("/isaaclab/video/auto_start_kit", self._video_auto_start_kit)
 
         # set setting to indicate no RTX sensors are used (set to True when RTX sensor is created)
         settings.set_bool("/isaaclab/render/rtx_sensors", False)

source/isaaclab/isaaclab/envs/direct_marl_env.py

@@ -151,8 +151,8 @@ def _init_sim(self, render_mode: str | None = None, **kwargs):
         # FIXME: This needs to be fixed in the future when we unify the UI functionalities even for
         # non-rendering modes.
         # Initialize when GUI is available OR when visualizers are active (headless rendering)
-        # Visualizers support camera updates via sim.set_camera_view() which forwards to all active visualizers
-        has_visualizers = bool(self.sim.get_setting("/isaaclab/visualizer"))
+        # ViewerCfg camera updates are applied through renderer camera control.
+        has_visualizers = self.sim.has_active_visualizers()
         if self.sim.has_gui or has_visualizers:
             self.viewport_camera_controller = ViewportCameraController(self, self.cfg.viewer)
         else:
@@ -522,7 +522,7 @@ def render(self, recompute: bool = False) -> np.ndarray | None:
             return None
         elif self.render_mode == "rgb_array":
             # check that if any render could have happened
-            if not self.sim.has_gui and not self.sim.has_offscreen_render:
+            if not self.sim.can_render_rgb_array():
                 raise RuntimeError(
                     f"Cannot render '{self.render_mode}' - no GUI and offscreen rendering not enabled."
                     " If running headless, make sure --enable_cameras is set."
@@ -531,10 +531,9 @@ def render(self, recompute: bool = False) -> np.ndarray | None:
             if not hasattr(self, "_rgb_annotator"):
                 import omni.replicator.core as rep
 
+                camera_prim_path = self.cfg.viewer.cam_prim_path
                 # create render product
-                self._render_product = rep.create.render_product(
-                    self.cfg.viewer.cam_prim_path, self.cfg.viewer.resolution
-                )
+                self._render_product = rep.create.render_product(camera_prim_path, self.cfg.viewer.resolution)
                 # create rgb annotator -- used to read data from the render product
                 self._rgb_annotator = rep.AnnotatorRegistry.get_annotator("rgb", device="cpu")
                 self._rgb_annotator.attach([self._render_product])

source/isaaclab/isaaclab/envs/direct_rl_env.py

@@ -156,8 +156,8 @@ def _init_sim(self, render_mode: str | None = None, **kwargs):
         # FIXME: This needs to be fixed in the future when we unify the UI functionalities even for
         # non-rendering modes.
         # Initialize when GUI is available OR when visualizers are active (headless rendering)
-        # Visualizers support camera updates via sim.set_camera_view() which forwards to all active visualizers
-        has_visualizers = bool(self.sim.get_setting("/isaaclab/visualizer"))
+        # ViewerCfg camera updates are applied through renderer camera control.
+        has_visualizers = self.sim.has_active_visualizers()
         if self.sim.has_gui or has_visualizers:
             self.viewport_camera_controller = ViewportCameraController(self, self.cfg.viewer)
         else:
@@ -490,7 +490,7 @@ def render(self, recompute: bool = False) -> np.ndarray | None:
             return None
         elif self.render_mode == "rgb_array":
             # check that if any render could have happened
-            if not self.sim.has_gui and not self.sim.has_offscreen_render:
+            if not self.sim.can_render_rgb_array():
                 raise RuntimeError(
                     f"Cannot render '{self.render_mode}' - no GUI and offscreen rendering not enabled."
                     " If running headless, make sure --enable_cameras is set."
@@ -499,10 +499,9 @@ def render(self, recompute: bool = False) -> np.ndarray | None:
             if not hasattr(self, "_rgb_annotator"):
                 import omni.replicator.core as rep
 
+                camera_prim_path = self.cfg.viewer.cam_prim_path
                 # create render product
-                self._render_product = rep.create.render_product(
-                    self.cfg.viewer.cam_prim_path, self.cfg.viewer.resolution
-                )
+                self._render_product = rep.create.render_product(camera_prim_path, self.cfg.viewer.resolution)
                 # create rgb annotator -- used to read data from the render product
                 self._rgb_annotator = rep.AnnotatorRegistry.get_annotator("rgb", device="cpu")
                 self._rgb_annotator.attach([self._render_product])

source/isaaclab/isaaclab/envs/manager_based_env.py

@@ -166,8 +166,8 @@ def _init_sim(self):
         # FIXME: This needs to be fixed in the future when we unify the UI functionalities even for
         # non-rendering modes.
         # Initialize when GUI is available OR when visualizers are active (headless rendering)
-        # Visualizers support camera updates via sim.set_camera_view() which forwards to all active visualizers
-        has_visualizers = bool(self.sim.get_setting("/isaaclab/visualizer"))
+        # ViewerCfg camera updates are applied through renderer camera control.
+        has_visualizers = self.sim.has_active_visualizers()
         if self.sim.has_gui or has_visualizers:
             self.viewport_camera_controller = ViewportCameraController(self, self.cfg.viewer)
         else:

source/isaaclab/isaaclab/envs/manager_based_rl_env.py

@@ -6,6 +6,7 @@
 # needed to import for allowing type-hinting: np.ndarray | None
 from __future__ import annotations
 
+import logging
 import math
 from collections.abc import Sequence
 from typing import Any, ClassVar
@@ -21,6 +22,8 @@
 from .manager_based_env import ManagerBasedEnv
 from .manager_based_rl_env_cfg import ManagerBasedRLEnvCfg
 
+logger = logging.getLogger(__name__)
+
 
 class ManagerBasedRLEnv(ManagerBasedEnv, gym.Env):
     """The superclass for the manager-based workflow reinforcement learning-based environments.
@@ -272,8 +275,7 @@ def render(self, recompute: bool = False) -> np.ndarray | None:
         elif self.render_mode == "rgb_array":
             # check that if any render could have happened
             # Check for GUI, offscreen rendering, or visualizers
-            has_visualizers = bool(self.sim.get_setting("/isaaclab/visualizer"))
-            if not (self.sim.has_gui or self.sim.has_offscreen_render or has_visualizers):
+            if not self.sim.can_render_rgb_array():
                 raise RuntimeError(
                     f"Cannot render '{self.render_mode}' - no GUI and offscreen rendering not enabled."
                     " If running headless, make sure --enable_cameras is set."
@@ -282,10 +284,9 @@ def render(self, recompute: bool = False) -> np.ndarray | None:
             if not hasattr(self, "_rgb_annotator"):
                 import omni.replicator.core as rep
 
+                camera_prim_path = self.cfg.viewer.cam_prim_path
                 # create render product
-                self._render_product = rep.create.render_product(
-                    self.cfg.viewer.cam_prim_path, self.cfg.viewer.resolution
-                )
+                self._render_product = rep.create.render_product(camera_prim_path, self.cfg.viewer.resolution)
                 # create rgb annotator -- used to read data from the render product
                 self._rgb_annotator = rep.AnnotatorRegistry.get_annotator("rgb", device="cpu")
                 self._rgb_annotator.attach([self._render_product])

source/isaaclab/isaaclab/envs/ui/viewport_camera_controller.py

@@ -218,8 +218,8 @@ def update_view_location(self, eye: Sequence[float] | None = None, lookat: Seque
         cam_eye = viewer_origin + self.default_cam_eye
         cam_target = viewer_origin + self.default_cam_lookat
 
-        # set the camera view
-        self._env.sim.set_camera_view(eye=cam_eye, target=cam_target)
+        # set the renderer viewport camera view (does not broadcast to visualizers)
+        self._env.sim.set_renderer_camera_view(eye=cam_eye, target=cam_target, camera_prim_path=self.cfg.cam_prim_path)
 
     """
     Private Functions

source/isaaclab/isaaclab/sim/simulation_context.py

@@ -180,14 +180,20 @@ def __init__(self, cfg: SimulationCfg | None = None):
         self._has_gui = bool(self.get_setting("/isaaclab/has_gui"))
         self._has_offscreen_render = bool(self.get_setting("/isaaclab/render/offscreen"))
         self._xr_enabled = bool(self.get_setting("/isaaclab/xr/enabled"))
+        self._video_auto_start_kit = bool(self.get_setting("/isaaclab/video/auto_start_kit"))
         # Note: has_rtx_sensors is NOT cached because it changes when Camera sensors are created
+        self._pending_camera_view: tuple[tuple[float, float, float], tuple[float, float, float]] | None = None
 
         # Simulation state
         self._is_playing = False
         self._is_stopped = True
 
         # Monotonic physics-step counter used by camera sensors for
         self._physics_step_count: int = 0
+        # Monotonic render-generation counter. This increments whenever render()
+        # is executed and lets downstream camera freshness logic distinguish
+        # render/reset transitions that occur without advancing physics steps.
+        self._render_generation: int = 0
 
         type(self)._instance = self  # Mark as valid singleton only after successful init
 
@@ -337,6 +343,16 @@ def has_offscreen_render(self) -> bool:
         """Returns whether offscreen rendering is enabled (cached at init)."""
         return self._has_offscreen_render
 
+    def has_active_visualizers(self) -> bool:
+        """Return whether any visualizer path is active for rendering/camera control."""
+        return bool(self.get_setting("/isaaclab/visualizer/types")) or bool(
+            self.get_setting("/isaaclab/video/auto_start_kit")
+        )
+
+    def can_render_rgb_array(self) -> bool:
+        """Return whether rgb-array rendering is currently available."""
+        return self.has_gui or self.has_offscreen_render or self.has_active_visualizers()
+
     @property
     def is_rendering(self) -> bool:
         """Returns whether rendering is active (GUI, RTX sensors, visualizers, or XR)."""
@@ -352,6 +368,11 @@ def get_physics_dt(self) -> float:
         """Returns the physics time step."""
         return self.physics_manager.get_physics_dt()
 
+    @property
+    def render_generation(self) -> int:
+        """Returns a monotonic counter for render() executions."""
+        return self._render_generation
+
     def _create_default_visualizer_configs(self, requested_visualizers: list[str]) -> list:
         """Create default visualizer configs for requested types.
 
@@ -534,6 +555,26 @@ def _resolve_visualizer_cfgs(self) -> list[Any]:
                         exc,
                     )
 
+        # Headless video auto-start: inject a KitVisualizer when needed so that
+        # app.update() is pumped and viewer camera updates can be applied in
+        # rgb-array recording flows.
+        if self._video_auto_start_kit and not cli_disable_all:
+            has_kit = any(getattr(cfg, "visualizer_type", None) == "kit" for cfg in resolved)
+            if not has_kit:
+                try:
+                    import importlib
+
+                    mod = importlib.import_module("isaaclab_visualizers.kit")
+                    kit_cfg_cls = getattr(mod, "KitVisualizerCfg")
+                    resolved.append(kit_cfg_cls(headless=True))
+                    logger.info("[SimulationContext] Auto-injecting KitVisualizer for headless video recording.")
+                except (ImportError, ModuleNotFoundError, AttributeError, TypeError) as exc:
+                    logger.warning(
+                        "[SimulationContext] Headless video could not auto-inject a KitVisualizer: %s. "
+                        "Install isaaclab_visualizers[kit] or pass --visualizer kit.",
+                        exc,
+                    )
+
         return resolved
 
     def initialize_visualizers(self) -> None:
@@ -577,6 +618,12 @@ def initialize_visualizers(self) -> None:
                     exc,
                 )
 
+        # Replay any camera pose requested before visualizers were initialized.
+        if self._pending_camera_view is not None:
+            eye, target = self._pending_camera_view
+            for viz in self._visualizers:
+                viz.set_camera_view(eye, target)
+
         if not self._visualizers and self._scene_data_provider is not None:
             close_provider = getattr(self._scene_data_provider, "close", None)
             if callable(close_provider):
@@ -627,9 +674,29 @@ def get_rendering_dt(self) -> float:
 
     def set_camera_view(self, eye: tuple, target: tuple) -> None:
         """Set camera view on all visualizers that support it."""
+        self._pending_camera_view = (tuple(eye), tuple(target))
         for viz in self._visualizers:
             viz.set_camera_view(eye, target)
 
+    def set_renderer_camera_view(
+        self,
+        eye: tuple[float, float, float] | list[float],
+        target: tuple[float, float, float] | list[float],
+        camera_prim_path: str = "/OmniverseKit_Persp",
+    ) -> None:
+        """Set camera view for renderer/viewport camera only.
+
+        This does not broadcast to visualizers.
+        """
+        try:
+            import isaacsim.core.utils.viewports as isaacsim_viewports
+
+            isaacsim_viewports.set_camera_view(
+                eye=list(eye), target=list(target), camera_prim_path=str(camera_prim_path)
+            )
+        except Exception as exc:
+            logger.debug("[SimulationContext] Renderer camera update skipped: %s", exc)
+
     def forward(self) -> None:
         """Update kinematics without stepping physics."""
         self.physics_manager.forward()
@@ -671,6 +738,7 @@ def render(self, mode: int | None = None) -> None:
         """
         self.physics_manager.pre_render()
         self.update_visualizers(self.get_rendering_dt())
+        self._render_generation += 1
 
         # Call render callbacks
         if hasattr(self, "_render_callbacks"):
@@ -695,6 +763,9 @@ def update_visualizers(self, dt: float) -> None:
                     visualizers_to_remove.append(viz)
                     continue
                 if viz.is_rendering_paused():
+                    # Keep visualizer event loops responsive while rendering is paused
+                    # so UI controls (e.g. "Resume Rendering") remain interactive.
+                    viz.step(0.0)
                     continue
                 while viz.is_training_paused() and viz.is_running():
                     viz.step(0.0)

source/isaaclab/isaaclab/visualizers/base_visualizer.py

@@ -187,6 +187,15 @@ def set_camera_view(self, eye: tuple, target: tuple) -> None:
         """
         pass
 
+    def _resolve_cfg_camera_pose(
+        self, visualizer_name: str
+    ) -> tuple[tuple[float, float, float], tuple[float, float, float]]:
+        """Resolve camera pose from cfg eye/lookat fields."""
+        del visualizer_name
+        eye = tuple(float(v) for v in getattr(self.cfg, "eye"))
+        lookat = tuple(float(v) for v in getattr(self.cfg, "lookat"))
+        return eye, lookat
+
     def _resolve_camera_pose_from_usd_path(
         self, usd_path: str
     ) -> tuple[tuple[float, float, float], tuple[float, float, float]] | None: