diff --git a/.github/workflows/license-exceptions.json b/.github/workflows/license-exceptions.json index 46143932838..49b1bb44a35 100644 --- a/.github/workflows/license-exceptions.json +++ b/.github/workflows/license-exceptions.json @@ -502,5 +502,10 @@ "package": "decorator", "license": "UNKNOWN", "comment": "BSD-2" + }, + { + "package": "pytetwild", + "license": "Mozilla Public License 2.0 (MPL 2.0)", + "comment": "MPL-2.0 / OSRB" } ] diff --git a/docs/conf.py b/docs/conf.py index 66a441be971..680ae165513 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -224,6 +224,7 @@ "hydra.core", "hydra.core.config_store", "omegaconf", + "newton", ] # List of zero or more Sphinx-specific warning categories to be squelched (i.e., diff --git a/docs/source/_static/css/custom.css b/docs/source/_static/css/custom.css index 7f4f1601c6c..e9b6b0bb603 100644 --- a/docs/source/_static/css/custom.css +++ b/docs/source/_static/css/custom.css @@ -110,3 +110,10 @@ a { text-align: center; vertical-align: middle; } + +.square-crop-figure img { + aspect-ratio: 1 / 1; + object-fit: cover; + object-position: center; + width: min(100%, 480px); +} diff --git a/docs/source/_static/newton/franka-mjwarp-vbd-coupling.png b/docs/source/_static/newton/franka-mjwarp-vbd-coupling.png new file mode 100644 index 00000000000..9066a3c65c7 Binary files /dev/null and b/docs/source/_static/newton/franka-mjwarp-vbd-coupling.png differ diff --git a/docs/source/api/index.rst b/docs/source/api/index.rst index 43ee461e124..2ff0cf6174b 100644 --- a/docs/source/api/index.rst +++ b/docs/source/api/index.rst @@ -25,6 +25,7 @@ The following modules are available in the ``isaaclab`` extension: physics renderers scene + scene_data sensors sim terrains @@ -83,6 +84,7 @@ The following modules are available in the ``isaaclab_contrib`` extension: actuators assets controllers + deformable mdp rl sensors @@ -129,17 +131,10 @@ The following modules are available in the ``isaaclab_physx`` extension: physics renderers sensors - sim.schemas - sim.spawners .. toctree:: :hidden: - lab_physx/isaaclab_physx.assets - lab_physx/isaaclab_physx.cloner - lab_physx/isaaclab_physx.physics - lab_physx/isaaclab_physx.renderers - lab_physx/isaaclab_physx.sensors lab_physx/isaaclab_physx.sim.schemas lab_physx/isaaclab_physx.sim.spawners @@ -163,12 +158,7 @@ The following modules are available in the ``isaaclab_newton`` extension: .. toctree:: :hidden: - lab_newton/isaaclab_newton.assets - lab_newton/isaaclab_newton.cloner - lab_newton/isaaclab_newton.physics - lab_newton/isaaclab_newton.renderers - lab_newton/isaaclab_newton.sensors - lab_newton/isaaclab_newton.sim.schemas + lab_newton/isaaclab_newton.sim.spawners isaaclab_ov extension --------------------- @@ -182,11 +172,6 @@ The following modules are available in the ``isaaclab_ov`` extension: renderers -.. toctree:: - :hidden: - - lab_ov/isaaclab_ov.renderers - isaaclab_assets extension ------------------------- @@ -200,12 +185,6 @@ The following modules are available in the ``isaaclab_assets`` extension: robots sensors -.. toctree:: - :hidden: - - lab_assets/isaaclab_assets.robots - lab_assets/isaaclab_assets.sensors - isaaclab_visualizers extension ------------------------------ @@ -221,14 +200,6 @@ The following modules are available in the ``isaaclab_visualizers`` extension: rerun viser -.. toctree:: - :hidden: - - lab_visualizers/isaaclab_visualizers.kit - lab_visualizers/isaaclab_visualizers.newton - lab_visualizers/isaaclab_visualizers.rerun - lab_visualizers/isaaclab_visualizers.viser - isaaclab_ovphysx extension --------------------------- @@ -244,13 +215,6 @@ The following modules are available in the ``isaaclab_ovphysx`` extension: cloner physics -.. toctree:: - :hidden: - - lab_ovphysx/isaaclab_ovphysx.assets - lab_ovphysx/isaaclab_ovphysx.cloner - lab_ovphysx/isaaclab_ovphysx.physics - isaaclab_experimental extension -------------------------------- @@ -266,13 +230,6 @@ The following modules are available in the ``isaaclab_experimental`` extension: managers utils -.. toctree:: - :hidden: - - lab_experimental/isaaclab_experimental.envs - lab_experimental/isaaclab_experimental.managers - lab_experimental/isaaclab_experimental.utils - isaaclab_tasks_experimental extension -------------------------------------- diff --git a/docs/source/api/lab/isaaclab.assets.rst b/docs/source/api/lab/isaaclab.assets.rst index 28356330b60..b7b980c2423 100644 --- a/docs/source/api/lab/isaaclab.assets.rst +++ b/docs/source/api/lab/isaaclab.assets.rst @@ -15,6 +15,11 @@ RigidObjectCollection RigidObjectCollectionData RigidObjectCollectionCfg + BaseDeformableObject + BaseDeformableObjectData + DeformableObject + DeformableObjectData + DeformableObjectCfg Articulation ArticulationData ArticulationCfg @@ -71,6 +76,37 @@ Rigid Object Collection :show-inheritance: :exclude-members: __init__, class_type +Deformable Object +----------------- + +.. autoclass:: DeformableObject + :members: + :inherited-members: + :show-inheritance: + +.. autoclass:: BaseDeformableObject + :members: + :inherited-members: + :show-inheritance: + +.. autoclass:: DeformableObjectData + :members: + :inherited-members: + :show-inheritance: + :exclude-members: __init__ + +.. autoclass:: BaseDeformableObjectData + :members: + :inherited-members: + :show-inheritance: + :exclude-members: __init__ + +.. autoclass:: DeformableObjectCfg + :members: + :inherited-members: + :show-inheritance: + :exclude-members: __init__, class_type, InitialStateCfg + Articulation ------------ diff --git a/docs/source/api/lab/isaaclab.scene_data.rst b/docs/source/api/lab/isaaclab.scene_data.rst new file mode 100644 index 00000000000..c36b7666861 --- /dev/null +++ b/docs/source/api/lab/isaaclab.scene_data.rst @@ -0,0 +1,33 @@ +isaaclab.scene_data +=================== + +.. automodule:: isaaclab.scene_data + + .. rubric:: Classes + + .. autosummary:: + + SceneDataProvider + SceneDataBackend + SceneDataFormat + +Scene Data Provider +------------------- + +.. autoclass:: SceneDataProvider + :members: + :undoc-members: + :show-inheritance: + +Scene Data Backend +------------------ + +.. autoclass:: SceneDataBackend + :members: + :undoc-members: + :show-inheritance: + +.. autoclass:: SceneDataFormat + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/lab/isaaclab.sim.schemas.rst b/docs/source/api/lab/isaaclab.sim.schemas.rst index bb0651c83ec..ea29860cbe4 100644 --- a/docs/source/api/lab/isaaclab.sim.schemas.rst +++ b/docs/source/api/lab/isaaclab.sim.schemas.rst @@ -19,6 +19,9 @@ isaaclab.sim.schemas JointDriveBaseCfg MeshCollisionBaseCfg MassPropertiesCfg + JointDrivePropertiesCfg + FixedTendonPropertiesCfg + DeformableBodyPropertiesBaseCfg .. rubric:: Mesh collision approximations (USD-only, no PhysX schema) @@ -44,9 +47,8 @@ isaaclab.sim.schemas define_mesh_collision_properties modify_mesh_collision_properties modify_fixed_tendon_properties - modify_spatial_tendon_properties - -.. currentmodule:: isaaclab.sim.schemas + define_deformable_body_properties + modify_deformable_body_properties Articulation Root ----------------- @@ -158,11 +160,10 @@ Tendon cfg classes are PhysX-only and live in Deformable Body --------------- -.. note:: - - Deformable body schemas have moved to the PhysX backend extension. See - :class:`isaaclab_physx.sim.schemas.DeformableBodyPropertiesCfg`, - :func:`isaaclab_physx.sim.schemas.define_deformable_body_properties`, and - :func:`isaaclab_physx.sim.schemas.modify_deformable_body_properties`. +.. autoclass:: DeformableBodyPropertiesBaseCfg + :members: + :show-inheritance: + :exclude-members: __init__ - For migration details, see :ref:`migrating-deformables`. +.. autofunction:: define_deformable_body_properties +.. autofunction:: modify_deformable_body_properties diff --git a/docs/source/api/lab/isaaclab.sim.spawners.rst b/docs/source/api/lab/isaaclab.sim.spawners.rst index a31968edfe0..d42fa13e524 100644 --- a/docs/source/api/lab/isaaclab.sim.spawners.rst +++ b/docs/source/api/lab/isaaclab.sim.spawners.rst @@ -21,6 +21,7 @@ SpawnerCfg RigidObjectSpawnerCfg + DeformableObjectSpawnerCfg Spawners -------- @@ -34,12 +35,10 @@ Spawners :show-inheritance: :exclude-members: __init__ -.. note:: - - ``DeformableObjectSpawnerCfg`` has moved to the PhysX backend extension. See - :class:`isaaclab_physx.sim.spawners.DeformableObjectSpawnerCfg`. - - For migration details, see :ref:`migrating-deformables`. +.. autoclass:: DeformableObjectSpawnerCfg + :members: + :show-inheritance: + :exclude-members: __init__ Shapes ------ @@ -110,6 +109,7 @@ Meshes MeshConeCfg MeshCuboidCfg MeshCylinderCfg + MeshRectangleCfg MeshSphereCfg .. autoclass:: MeshCfg @@ -144,6 +144,13 @@ Meshes :show-inheritance: :exclude-members: __init__, func +.. autofunction:: spawn_mesh_rectangle + +.. autoclass:: MeshRectangleCfg + :members: + :show-inheritance: + :exclude-members: __init__, func + .. autofunction:: spawn_mesh_sphere .. autoclass:: MeshSphereCfg @@ -261,6 +268,10 @@ Materials GlassMdlCfg PhysicsMaterialCfg RigidBodyMaterialCfg + DeformableBodyMaterialBaseCfg + SurfaceDeformableBodyMaterialBaseCfg + DeformableBodyMaterialCfg + SurfaceDeformableBodyMaterialCfg Visual Materials ~~~~~~~~~~~~~~~~ @@ -298,15 +309,36 @@ Physical Materials :members: :exclude-members: __init__, func +.. autofunction:: spawn_deformable_body_material + +.. autoclass:: DeformableBodyMaterialBaseCfg + :members: + :show-inheritance: + :exclude-members: __init__, func + +.. autoclass:: SurfaceDeformableBodyMaterialBaseCfg + :members: + :show-inheritance: + :exclude-members: __init__, func + .. note:: - ``DeformableBodyMaterialCfg``, ``SurfaceDeformableBodyMaterialCfg``, and - ``spawn_deformable_body_material`` have moved to the PhysX backend extension. See - :class:`isaaclab_physx.sim.spawners.materials.DeformableBodyMaterialCfg`, - :class:`isaaclab_physx.sim.spawners.materials.SurfaceDeformableBodyMaterialCfg`, and - :func:`isaaclab_physx.sim.spawners.materials.spawn_deformable_body_material`. + Backend-specific deformable material cfgs live in + :mod:`isaaclab_physx.sim.spawners.materials` and + :mod:`isaaclab_newton.sim.spawners.materials`. The legacy default names below + are forwarded to the deprecated PhysX aliases for compatibility. - For migration details, see :ref:`migrating-deformables`. +.. autoclass:: DeformableBodyMaterialCfg + :no-index: + :members: + :show-inheritance: + :exclude-members: __init__, func + +.. autoclass:: SurfaceDeformableBodyMaterialCfg + :no-index: + :members: + :show-inheritance: + :exclude-members: __init__, func Wrappers -------- diff --git a/docs/source/api/lab_contrib/isaaclab_contrib.deformable.rst b/docs/source/api/lab_contrib/isaaclab_contrib.deformable.rst new file mode 100644 index 00000000000..0267a922c5c --- /dev/null +++ b/docs/source/api/lab_contrib/isaaclab_contrib.deformable.rst @@ -0,0 +1,73 @@ +isaaclab_contrib.deformable +=========================== + +.. automodule:: isaaclab_contrib.deformable + + .. rubric:: Classes + + .. autosummary:: + + deformable_object.DeformableObject + deformable_object_data.DeformableObjectData + newton_manager_cfg.VBDSolverCfg + newton_manager_cfg.CoupledMJWarpVBDSolverCfg + newton_manager_cfg.CoupledFeatherstoneVBDSolverCfg + newton_manager_cfg.NewtonModelCfg + vbd_manager.NewtonVBDManager + coupled_mjwarp_vbd_manager.NewtonCoupledMJWarpVBDManager + coupled_featherstone_vbd_manager.NewtonCoupledFeatherstoneVBDManager + +Deformable Object +----------------- + +.. autoclass:: isaaclab_contrib.deformable.deformable_object.DeformableObject + :members: + :inherited-members: + :show-inheritance: + +.. autoclass:: isaaclab_contrib.deformable.deformable_object_data.DeformableObjectData + :members: + :inherited-members: + :show-inheritance: + :exclude-members: __init__ + +Newton Solver Configurations +---------------------------- + +.. autoclass:: isaaclab_contrib.deformable.newton_manager_cfg.VBDSolverCfg + :members: + :show-inheritance: + :exclude-members: __init__ + +.. autoclass:: isaaclab_contrib.deformable.newton_manager_cfg.CoupledMJWarpVBDSolverCfg + :members: + :show-inheritance: + :exclude-members: __init__ + +.. autoclass:: isaaclab_contrib.deformable.newton_manager_cfg.CoupledFeatherstoneVBDSolverCfg + :members: + :show-inheritance: + :exclude-members: __init__ + +.. autoclass:: isaaclab_contrib.deformable.newton_manager_cfg.NewtonModelCfg + :members: + :show-inheritance: + :exclude-members: __init__ + +Newton Solver Managers +---------------------- + +.. autoclass:: isaaclab_contrib.deformable.vbd_manager.NewtonVBDManager + :members: + :inherited-members: + :show-inheritance: + +.. autoclass:: isaaclab_contrib.deformable.coupled_mjwarp_vbd_manager.NewtonCoupledMJWarpVBDManager + :members: + :inherited-members: + :show-inheritance: + +.. autoclass:: isaaclab_contrib.deformable.coupled_featherstone_vbd_manager.NewtonCoupledFeatherstoneVBDManager + :members: + :inherited-members: + :show-inheritance: diff --git a/docs/source/api/lab_experimental/isaaclab_experimental.envs.rst b/docs/source/api/lab_experimental/isaaclab_experimental.envs.rst new file mode 100644 index 00000000000..6e87a9a17cb --- /dev/null +++ b/docs/source/api/lab_experimental/isaaclab_experimental.envs.rst @@ -0,0 +1,4 @@ +isaaclab\_experimental.envs +=========================== + +.. automodule:: isaaclab_experimental.envs diff --git a/docs/source/api/lab_experimental/isaaclab_experimental.managers.rst b/docs/source/api/lab_experimental/isaaclab_experimental.managers.rst new file mode 100644 index 00000000000..c0f56ed1a6a --- /dev/null +++ b/docs/source/api/lab_experimental/isaaclab_experimental.managers.rst @@ -0,0 +1,4 @@ +isaaclab\_experimental.managers +=============================== + +.. automodule:: isaaclab_experimental.managers diff --git a/docs/source/api/lab_experimental/isaaclab_experimental.utils.rst b/docs/source/api/lab_experimental/isaaclab_experimental.utils.rst new file mode 100644 index 00000000000..49d7bd6dcdb --- /dev/null +++ b/docs/source/api/lab_experimental/isaaclab_experimental.utils.rst @@ -0,0 +1,4 @@ +isaaclab\_experimental.utils +============================ + +.. automodule:: isaaclab_experimental.utils diff --git a/docs/source/api/lab_newton/isaaclab_newton.assets.rst b/docs/source/api/lab_newton/isaaclab_newton.assets.rst index 8dc6f9b8f58..f044f9b089f 100644 --- a/docs/source/api/lab_newton/isaaclab_newton.assets.rst +++ b/docs/source/api/lab_newton/isaaclab_newton.assets.rst @@ -2,3 +2,82 @@ ======================= .. automodule:: isaaclab_newton.assets + + .. rubric:: Classes + + .. autosummary:: + + Articulation + ArticulationData + RigidObject + RigidObjectData + RigidObjectCollection + RigidObjectCollectionData + DeformableObject + DeformableObjectData + +.. currentmodule:: isaaclab_newton.assets + +Articulation +------------ + +.. autoclass:: Articulation + :members: + :inherited-members: + :show-inheritance: + +.. autoclass:: ArticulationData + :members: + :inherited-members: + :show-inheritance: + :exclude-members: __init__ + +Rigid Object +------------ + +.. autoclass:: RigidObject + :members: + :inherited-members: + :show-inheritance: + +.. autoclass:: RigidObjectData + :members: + :inherited-members: + :show-inheritance: + :exclude-members: __init__ + +Rigid Object Collection +----------------------- + +.. autoclass:: RigidObjectCollection + :members: + :inherited-members: + :show-inheritance: + +.. autoclass:: RigidObjectCollectionData + :members: + :inherited-members: + :show-inheritance: + :exclude-members: __init__ + +Deformable Object +----------------- + +.. note:: + + :class:`isaaclab.assets.DeformableObjectCfg` is the shared configuration + class for deformable objects. The Newton extension exposes the Newton + implementation of :class:`isaaclab.assets.DeformableObject`, while + deformable schema and material cfgs referenced by ``spawn`` remain + backend-specific. + +.. autoclass:: DeformableObject + :members: + :inherited-members: + :show-inheritance: + +.. autoclass:: DeformableObjectData + :members: + :inherited-members: + :show-inheritance: + :exclude-members: __init__ diff --git a/docs/source/api/lab_newton/isaaclab_newton.physics.rst b/docs/source/api/lab_newton/isaaclab_newton.physics.rst index 692293ce88e..b452f3ad538 100644 --- a/docs/source/api/lab_newton/isaaclab_newton.physics.rst +++ b/docs/source/api/lab_newton/isaaclab_newton.physics.rst @@ -2,3 +2,103 @@ ======================== .. automodule:: isaaclab_newton.physics + + .. rubric:: Classes + + .. autosummary:: + + NewtonManager + NewtonCfg + NewtonSolverCfg + MJWarpSolverCfg + XPBDSolverCfg + FeatherstoneSolverCfg + KaminoSolverCfg + NewtonCollisionPipelineCfg + HydroelasticSDFCfg + NewtonShapeCfg + NewtonMJWarpManager + NewtonXPBDManager + NewtonFeatherstoneManager + NewtonKaminoManager + +.. currentmodule:: isaaclab_newton.physics + +Physics Manager +--------------- + +.. autoclass:: NewtonManager + :members: + :inherited-members: + :show-inheritance: + +Physics Configuration +--------------------- + +.. autoclass:: NewtonCfg + :members: + :show-inheritance: + :exclude-members: __init__ + +.. autoclass:: NewtonSolverCfg + :members: + :show-inheritance: + :exclude-members: __init__ + +.. autoclass:: MJWarpSolverCfg + :members: + :show-inheritance: + :exclude-members: __init__ + +.. autoclass:: XPBDSolverCfg + :members: + :show-inheritance: + :exclude-members: __init__ + +.. autoclass:: FeatherstoneSolverCfg + :members: + :show-inheritance: + :exclude-members: __init__ + +.. autoclass:: KaminoSolverCfg + :members: + :show-inheritance: + :exclude-members: __init__ + +.. autoclass:: NewtonCollisionPipelineCfg + :members: + :show-inheritance: + :exclude-members: __init__ + +.. autoclass:: HydroelasticSDFCfg + :members: + :show-inheritance: + :exclude-members: __init__ + +.. autoclass:: NewtonShapeCfg + :members: + :show-inheritance: + :exclude-members: __init__ + +Solver Managers +--------------- + +.. autoclass:: NewtonMJWarpManager + :members: + :inherited-members: + :show-inheritance: + +.. autoclass:: NewtonXPBDManager + :members: + :inherited-members: + :show-inheritance: + +.. autoclass:: NewtonFeatherstoneManager + :members: + :inherited-members: + :show-inheritance: + +.. autoclass:: NewtonKaminoManager + :members: + :inherited-members: + :show-inheritance: diff --git a/docs/source/api/lab_newton/isaaclab_newton.sim.schemas.rst b/docs/source/api/lab_newton/isaaclab_newton.sim.schemas.rst index f0dde258fe8..78fb53b7098 100644 --- a/docs/source/api/lab_newton/isaaclab_newton.sim.schemas.rst +++ b/docs/source/api/lab_newton/isaaclab_newton.sim.schemas.rst @@ -16,6 +16,7 @@ isaaclab_newton.sim.schemas .. autosummary:: + NewtonDeformableBodyPropertiesCfg NewtonRigidBodyPropertiesCfg NewtonJointDrivePropertiesCfg NewtonCollisionPropertiesCfg @@ -32,6 +33,17 @@ isaaclab_newton.sim.schemas .. currentmodule:: isaaclab_newton.sim.schemas +Deformable Body +--------------- + +.. autoclass:: NewtonDeformableBodyPropertiesCfg + :members: + :show-inheritance: + :exclude-members: __init__ + +Schema define and modify functions remain unified in +:mod:`isaaclab.sim.schemas`. + Rigid Body ---------- diff --git a/docs/source/api/lab_newton/isaaclab_newton.sim.spawners.rst b/docs/source/api/lab_newton/isaaclab_newton.sim.spawners.rst new file mode 100644 index 00000000000..d60b953ce5c --- /dev/null +++ b/docs/source/api/lab_newton/isaaclab_newton.sim.spawners.rst @@ -0,0 +1,33 @@ +isaaclab_newton.sim.spawners +============================ + +.. automodule:: isaaclab_newton.sim.spawners.materials + + .. rubric:: Classes + + .. autosummary:: + + NewtonDeformableBodyMaterialCfg + NewtonDeformableMaterialCfg + NewtonSurfaceDeformableBodyMaterialCfg + +Deformable Materials +-------------------- + +Newton provides the backend-specific deformable material cfgs. Deformable material spawning is unified in +:func:`isaaclab.sim.spawners.materials.spawn_deformable_body_material`. + +.. autoclass:: NewtonDeformableBodyMaterialCfg + :members: + :show-inheritance: + :exclude-members: __init__, func + +.. autoclass:: NewtonDeformableMaterialCfg + :members: + :show-inheritance: + :exclude-members: __init__, func + +.. autoclass:: NewtonSurfaceDeformableBodyMaterialCfg + :members: + :show-inheritance: + :exclude-members: __init__, func diff --git a/docs/source/api/lab_ovphysx/isaaclab_ovphysx.assets.rst b/docs/source/api/lab_ovphysx/isaaclab_ovphysx.assets.rst new file mode 100644 index 00000000000..165c12772c0 --- /dev/null +++ b/docs/source/api/lab_ovphysx/isaaclab_ovphysx.assets.rst @@ -0,0 +1,4 @@ +isaaclab\_ovphysx.assets +======================== + +.. automodule:: isaaclab_ovphysx.assets diff --git a/docs/source/api/lab_ovphysx/isaaclab_ovphysx.cloner.rst b/docs/source/api/lab_ovphysx/isaaclab_ovphysx.cloner.rst new file mode 100644 index 00000000000..7467364b075 --- /dev/null +++ b/docs/source/api/lab_ovphysx/isaaclab_ovphysx.cloner.rst @@ -0,0 +1,4 @@ +isaaclab\_ovphysx.cloner +======================== + +.. automodule:: isaaclab_ovphysx.cloner diff --git a/docs/source/api/lab_ovphysx/isaaclab_ovphysx.physics.rst b/docs/source/api/lab_ovphysx/isaaclab_ovphysx.physics.rst new file mode 100644 index 00000000000..aa8c37a4af5 --- /dev/null +++ b/docs/source/api/lab_ovphysx/isaaclab_ovphysx.physics.rst @@ -0,0 +1,4 @@ +isaaclab\_ovphysx.physics +========================= + +.. automodule:: isaaclab_ovphysx.physics diff --git a/docs/source/api/lab_physx/isaaclab_physx.assets.rst b/docs/source/api/lab_physx/isaaclab_physx.assets.rst index b631dcbcb7b..851a01f9b6e 100644 --- a/docs/source/api/lab_physx/isaaclab_physx.assets.rst +++ b/docs/source/api/lab_physx/isaaclab_physx.assets.rst @@ -16,7 +16,6 @@ RigidObjectCollectionData DeformableObject DeformableObjectData - DeformableObjectCfg SurfaceGripper SurfaceGripperCfg @@ -78,11 +77,13 @@ Deformable Object :show-inheritance: :exclude-members: __init__ -.. autoclass:: DeformableObjectCfg - :members: - :inherited-members: - :show-inheritance: - :exclude-members: __init__, class_type, InitialStateCfg +.. note:: + + :class:`isaaclab.assets.DeformableObjectCfg` is the shared configuration + class for deformable objects. The PhysX extension provides the PhysX + implementation of :class:`isaaclab.assets.DeformableObject`, while + deformable schema and material cfgs referenced by ``spawn`` remain + backend-specific. Surface Gripper --------------- diff --git a/docs/source/api/lab_physx/isaaclab_physx.sim.schemas.rst b/docs/source/api/lab_physx/isaaclab_physx.sim.schemas.rst index 30526906899..7a01585545d 100644 --- a/docs/source/api/lab_physx/isaaclab_physx.sim.schemas.rst +++ b/docs/source/api/lab_physx/isaaclab_physx.sim.schemas.rst @@ -49,6 +49,9 @@ isaaclab_physx.sim.schemas .. autosummary:: + OmniPhysicsDeformableBodyPropertiesCfg + PhysxDeformableCollisionPropertiesCfg + PhysxDeformableBodyPropertiesCfg DeformableBodyPropertiesCfg .. rubric:: Functions @@ -136,10 +139,25 @@ Tendon Deformable Body --------------- +.. autoclass:: OmniPhysicsDeformableBodyPropertiesCfg + :members: + :show-inheritance: + :exclude-members: __init__ + +.. autoclass:: PhysxDeformableCollisionPropertiesCfg + :members: + :show-inheritance: + :exclude-members: __init__ + +.. autoclass:: PhysxDeformableBodyPropertiesCfg + :members: + :show-inheritance: + :exclude-members: __init__ + .. autoclass:: DeformableBodyPropertiesCfg :members: :show-inheritance: :exclude-members: __init__ -.. autofunction:: define_deformable_body_properties -.. autofunction:: modify_deformable_body_properties +Schema define and modify functions remain unified in +:mod:`isaaclab.sim.schemas`. diff --git a/docs/source/api/lab_physx/isaaclab_physx.sim.spawners.rst b/docs/source/api/lab_physx/isaaclab_physx.sim.spawners.rst index 27463a44425..7a2ace2a33f 100644 --- a/docs/source/api/lab_physx/isaaclab_physx.sim.spawners.rst +++ b/docs/source/api/lab_physx/isaaclab_physx.sim.spawners.rst @@ -1,45 +1,41 @@ isaaclab_physx.sim.spawners =========================== -.. automodule:: isaaclab_physx.sim.spawners - - .. rubric:: Submodules - - .. autosummary:: - - materials +.. automodule:: isaaclab_physx.sim.spawners.materials .. rubric:: Classes .. autosummary:: - DeformableObjectSpawnerCfg + PhysxDeformableBodyMaterialCfg + PhysxSurfaceDeformableBodyMaterialCfg + PhysXDeformableMaterialCfg + DeformableBodyMaterialCfg + SurfaceDeformableBodyMaterialCfg -.. currentmodule:: isaaclab_physx.sim.spawners +Deformable Materials +-------------------- -Spawners --------- +PhysX provides the backend-specific deformable material cfgs. Deformable material spawning is unified in +:func:`isaaclab.sim.spawners.materials.spawn_deformable_body_material`. -.. autoclass:: DeformableObjectSpawnerCfg +.. autoclass:: PhysxDeformableBodyMaterialCfg :members: :show-inheritance: - :exclude-members: __init__ - -Materials ---------- - -.. automodule:: isaaclab_physx.sim.spawners.materials - - .. rubric:: Classes - - .. autosummary:: + :exclude-members: __init__, func - DeformableBodyMaterialCfg - SurfaceDeformableBodyMaterialCfg +.. autoclass:: PhysxSurfaceDeformableBodyMaterialCfg + :members: + :show-inheritance: + :exclude-members: __init__, func -.. currentmodule:: isaaclab_physx.sim.spawners.materials +.. autoclass:: PhysXDeformableMaterialCfg + :members: + :show-inheritance: + :exclude-members: __init__, func -.. autofunction:: spawn_deformable_body_material +Deprecated Aliases +------------------ .. autoclass:: DeformableBodyMaterialCfg :members: diff --git a/docs/source/features/visualization.rst b/docs/source/features/visualization.rst index c37fb6d9be9..60eb4e7c01c 100644 --- a/docs/source/features/visualization.rst +++ b/docs/source/features/visualization.rst @@ -22,10 +22,10 @@ Isaac Lab supports four visualizer backends, each optimized for different use ca - Key Features * - **Omniverse** - High-fidelity, Isaac Sim integration - - USD, visualization markers, live plots + - USD, visualization markers, live plots, tiled camera panel * - **Newton** - Fast iteration - - Low overhead, visualization markers + - Low overhead, visualization markers, tiled camera panel * - **Rerun** - Remote viewing, replay - Webviewer, time scrubbing, recording export, visualization markers @@ -121,6 +121,8 @@ You can also configure custom visualizers in the code by defining ``VisualizerCf ), ViserVisualizerCfg( port=8080, + bind_address="0.0.0.0", + display_address="localhost", share=False, ), ] @@ -185,6 +187,35 @@ Also, there is a CLI arg ``--max_visible_envs`` that overrides ``VisualizerCfg.m - any - Run headless; ``--headless`` takes precedence. +Camera Modes +~~~~~~~~~~~~ + +The default visualizer camera mode is interactive, with ``eye`` and ``lookat`` specifying the initial pose. +Kit and Newton visualizers can also run additional tiled camera image panels. +If ``tiled_cam_view=True`` is set, another window is launched in the visualizer which shows +a non-interactive tiled camera image view. + +Kit and Newton cap tiled camera views at 100 tiles. + +Note, Kit tiled camera views require launching with ``--enable_cameras``. + +.. list-table:: Camera configuration modes + :header-rows: 1 + :widths: 24 30 46 + + * - Mode + - Key fields + - Behavior + * - **Default interactive** + - ``tiled_cam_view=False``, ``eye=(4, -4, 3)``, ``lookat=(0, 0, 0)`` + - Interactive visualizer camera starts at ``eye`` and looks at the fixed ``lookat`` coordinate. + * - Generated tiled camera + - ``tiled_cam_view=True``, ``tiled_cam_prim_path=None``, ``tiled_cam_target_prim_path="/World/envs/*/Robot/base"`` + - The visualizer creates per-env cameras. Each camera looks at the matched target prim, with ``tiled_cam_eye`` as an offset from that target. + * - Existing tiled camera sensors + - ``tiled_cam_view=True``, ``tiled_cam_prim_path="/World/envs/*/Camera"`` + - The visualizer displays existing Isaac Lab ``Camera`` sensor output. Generated-camera fields such as ``tiled_cam_eye`` and ``tiled_cam_target_prim_path`` are ignored. + Video Recording --------------- @@ -287,6 +318,7 @@ Omniverse Visualizer - Live plots for monitoring training metrics - Full Isaac Sim rendering capabilities and tooling - Visualization markers for debugging (arrows, frames, object targets, etc.) +- Tiled camera views which can track multiple robots **Core Configuration:** @@ -309,7 +341,6 @@ Omniverse Visualizer enable_live_plots=True, ) - Newton Visualizer ~~~~~~~~~~~~~~~~~ @@ -320,6 +351,7 @@ Newton Visualizer - Adjustable update frequency for performance tuning - Some customizable rendering options (shadows, sky, wireframe) - Visualization markers (joints, contacts, springs, COM, debug markers) +- Tiled camera views which can track multiple robots **Interactive Controls:** @@ -357,6 +389,17 @@ Newton Visualizer # Camera settings eye=(8.0, 8.0, 3.0), # Initial camera position (x, y, z) lookat=(0.0, 0.0, 0.0), # Camera look-at target + focal_length=12.0, # Camera focal length in millimeters + + # Tiled camera view settings + tiled_cam_view=True, # Enable non-interactive tiled camera image view + tiled_cam_num=16, # Number of generated camera tiles to display + tiled_cam_env_indices=None, # Optional explicit env ids to show in the tiled view + tiled_cam_prim_path=None, # Existing Camera sensor prim path, e.g. "/World/envs/*/Camera" + tiled_cam_eye=(4.0, -4.0, 3.0), # Eye offset for generated tiled cameras + tiled_cam_target_prim_path=( # Prim that generated cameras follow/look at + "/World/envs/*/Robot/base" + ), # Performance tuning update_frequency=1, # Update every N frames (1=every frame) @@ -390,6 +433,22 @@ Rerun Visualizer - Timeline scrubbing and playback controls of recordings - Visualization debug markers +.. important:: + + A highlighted Rerun browser URL is printed in the logs before the main simulation or training loop begins. + Ctrl-click the printed URL in supported terminals/IDEs to open it. Set ``open_browser=True`` to automatically + open the browser tab instead. + + Example: + + .. code-block:: text + + ╭─────────────────────────── rerun (listening *:9090) ───────────────────────────╮ + │ ╷ │ + │ URL │ http://127.0.0.1:9090/?url=rerun%2Bhttp://127.0.0.1:9876/proxy │ + │ ╵ │ + ╰────────────────────────────────────────────────────────────────────────────────╯ + **Core Configuration:** .. code-block:: python @@ -400,8 +459,9 @@ Rerun Visualizer # Server settings app_id="isaaclab-simulation", # Application identifier for viewer grpc_port=9876, # gRPC endpoint for logging SDK connection - web_port=9090, # Port for local web viewer (launched in browser) + web_port=9090, # Port for local web viewer URL printed in logs bind_address="0.0.0.0", # Endpoint host formatting/reuse checks + open_browser=False, # Set True to auto-launch the browser # Camera settings eye=(8.0, 8.0, 3.0), # Initial camera position (x, y, z) @@ -427,7 +487,7 @@ The `Viser `_ visualizer provides a **web-based** 3D view simulations powered by the Newton Warp renderer. It streams the simulation state to a local web server, allowing you to view and interact with the scene from any browser. -**Key features:** +**Main Features:** - Browser-based visualization accessible at ``http://localhost:8080`` by default - Optional public share URL for remote viewing @@ -435,34 +495,55 @@ server, allowing you to view and interact with the scene from any browser. - Environment filtering to control which environments are rendered - Visualization debug markers -**Launch with Viser:** +.. important:: -.. code-block:: bash + A highlighted Viser browser URL is printed in the logs before the main simulation or training loop begins. + Ctrl-click the printed URL in supported terminals/IDEs to open it. Set ``open_browser=True`` to automatically + open the browser tab instead. For remote access, keep ``bind_address="0.0.0.0"`` and set + ``display_address`` to the hostname or IP address reachable from your browser. + + Example: - ./isaaclab.sh -p source/isaaclab_tasks/isaaclab_tasks/direct/cartpole/cartpole_env.py --viz viser + .. code-block:: text -**Configuration example:** + ╭────── viser (listening *:8080) ───────╮ + │ ╷ │ + │ URL │ http://localhost:8080 │ + │ ╵ │ + ╰───────────────────────────────────────╯ + +**Core Configuration:** .. code-block:: python from isaaclab_visualizers.viser import ViserVisualizerCfg visualizer_cfg = ViserVisualizerCfg( - port=8080, - open_browser=True, - label="Isaac Lab Simulation", - share=False, - max_visible_envs=16, - ) + # Server settings + port=8080, # Port for local Viser web server + bind_address="0.0.0.0", # Interface to listen on; use 0.0.0.0 for remote access + display_address="localhost", # Host/IP shown in the printed browser URL + open_browser=False, # Set True to auto-launch the browser + label="Isaac Lab Simulation", # Page title shown in the viewer + share=False, # Request a public share URL for remote viewing + verbose=True, # Print viewer server startup information -**Configuration options:** + # Camera settings + eye=(8.0, 8.0, 3.0), # Initial camera position (x, y, z) + lookat=(0.0, 0.0, 0.0), # Camera look-at target + + # Environment filtering + max_visible_envs=16, # Maximum number of environments to visualize + + # Recording + record_to_viser="recording.viser", # Path to save .viser file (None = no recording) + ) -- ``port`` (int, default ``8080``): Port of the local Viser web server. -- ``open_browser`` (bool, default ``True``): Automatically open the viewer URL in a browser. -- ``label`` (str or None, default ``"Isaac Lab Simulation"``): Page title shown in the viewer. -- ``share`` (bool, default ``False``): Request a public share URL from Viser for remote viewing. -- ``record_to_viser`` (str or None, default ``None``): Path to save a ``.viser`` recording file. -- ``verbose`` (bool, default ``True``): Print viewer server startup information. +Viser uses an in-process ``viser.ViserServer`` through ``newton.viewer.ViewerViser``. ``bind_address`` +controls the network interface that the server listens on, while ``display_address`` controls only the +URL printed by Isaac Lab. On a remote machine, set ``display_address`` to the machine hostname/IP and +ensure the configured ``port`` is reachable from your browser. Set ``share=True`` to request Viser's +public share/tunnel URL when that service is available. .. note:: @@ -507,7 +588,7 @@ Currently, live plots are only available in the Kit Visualizer. **Viser Visualizer Renderer Requirement** The Viser visualizer requires a Newton model, which is provided automatically by -:class:`~isaaclab.scene.scene_data_provider.SceneDataProvider` regardless of the active physics +:class:`~isaaclab.scene_data.SceneDataProvider` regardless of the active physics backend or renderer. It is compatible with all rendering backends (RTX, Newton Warp, OVRTX). diff --git a/docs/source/migration/migrating_deformables.rst b/docs/source/migration/migrating_deformables.rst index efb7f7da689..cce2e15fa3e 100644 --- a/docs/source/migration/migrating_deformables.rst +++ b/docs/source/migration/migrating_deformables.rst @@ -7,8 +7,11 @@ Migration of Deformables In the newer versions of Omni Physics (107.0 and later), the old deformable body functionality has become deprecated. The following sections describe the changes to migrate to the new Omni Physics API, specifically moving away from -Soft Bodies and towards Surface and Volume Deformables. We currently only support deformable bodies in the PhysX -backend, hence these features are implemented in ``isaaclab_physx``. +Soft Bodies and towards Surface and Volume Deformables. The deformable object asset classes remain in +``isaaclab.assets``. Schema define/modify functions remain unified in ``isaaclab.sim.schemas``, and deformable +material spawning remains unified in ``isaaclab.sim.spawners.materials``. Deformable property and material +configuration classes are backend-specific: PhysX configurations live in ``isaaclab_physx.sim`` and Newton +configurations live in ``isaaclab_newton.sim``. .. note:: @@ -31,8 +34,12 @@ With the new Omni Physics API, deformable bodies are split into two distinct typ The type of deformable is determined by the **physics material** assigned to the object: -- :class:`~isaaclab_physx.sim.DeformableBodyMaterialCfg` creates a **volume** deformable. -- :class:`~isaaclab_physx.sim.SurfaceDeformableBodyMaterialCfg` creates a **surface** deformable. +- :class:`~isaaclab_physx.sim.PhysxDeformableBodyMaterialCfg` creates a PhysX **volume** deformable. +- :class:`~isaaclab_physx.sim.PhysxSurfaceDeformableBodyMaterialCfg` creates a PhysX **surface** deformable. +- :class:`~isaaclab_newton.sim.spawners.materials.NewtonDeformableBodyMaterialCfg` creates a Newton + **volume** deformable. +- :class:`~isaaclab_newton.sim.spawners.materials.NewtonSurfaceDeformableBodyMaterialCfg` creates a Newton + **surface** deformable. Migration from the Old API @@ -41,8 +48,8 @@ Migration from the Old API Import Changes ^^^^^^^^^^^^^^ -All deformable-related classes have moved from ``isaaclab`` to ``isaaclab_physx``. The table below summarizes the -import changes: +Deformable object cfgs remain in ``isaaclab.assets``. Deformable schema and material cfgs should be imported +from the physics backend package: .. list-table:: :header-rows: 1 @@ -51,15 +58,26 @@ import changes: * - Old Import - New Import * - ``from isaaclab.sim import DeformableBodyPropertiesCfg`` - - ``from isaaclab_physx.sim import DeformableBodyPropertiesCfg`` + - ``from isaaclab_physx.sim import PhysxDeformableBodyPropertiesCfg`` * - ``from isaaclab.sim import DeformableBodyMaterialCfg`` - - ``from isaaclab_physx.sim import DeformableBodyMaterialCfg`` + - ``from isaaclab_physx.sim import PhysxDeformableBodyMaterialCfg`` + * - ``from isaaclab.sim import SurfaceDeformableBodyMaterialCfg`` + - ``from isaaclab_physx.sim import PhysxSurfaceDeformableBodyMaterialCfg`` + * - ``from isaaclab.sim import DeformableBodyPropertiesCfg`` + - ``from isaaclab_newton.sim.schemas import NewtonDeformableBodyPropertiesCfg`` + * - ``from isaaclab.sim import DeformableBodyMaterialCfg`` + - ``from isaaclab_newton.sim.spawners.materials import NewtonDeformableBodyMaterialCfg`` + * - ``from isaaclab.sim import SurfaceDeformableBodyMaterialCfg`` + - ``from isaaclab_newton.sim.spawners.materials import NewtonSurfaceDeformableBodyMaterialCfg`` + * - ``from isaaclab_physx.assets import DeformableObjectCfg`` + - ``from isaaclab.assets import DeformableObjectCfg`` Removed Properties ^^^^^^^^^^^^^^^^^^ -The following properties have been **removed** from :class:`~isaaclab_physx.sim.DeformableBodyPropertiesCfg`: +The following properties have been **removed** from +:class:`~isaaclab_physx.sim.PhysxDeformableBodyPropertiesCfg`: - ``collision_simplification`` and related parameters (``collision_simplification_remeshing``, ``collision_simplification_target_triangle_count``, ``collision_simplification_force_conforming``, @@ -75,8 +93,11 @@ The following properties have been **removed** from :class:`~isaaclab_physx.sim. Added Properties ^^^^^^^^^^^^^^^^ -The following properties have been **added** to :class:`~isaaclab_physx.sim.DeformableBodyPropertiesCfg`: +The following properties have been **added** to +:class:`~isaaclab_physx.sim.PhysxDeformableBodyPropertiesCfg`: +- ``deformable_body_enabled``, ``kinematic_enabled``, and ``mass`` — OmniPhysics + deformable body properties owned by the PhysX backend cfg. - ``linear_damping`` — linear damping coefficient [1/s]. - ``max_linear_velocity`` — maximum allowable linear velocity [m/s]. A negative value lets the simulation choose a per-vertex value dynamically (currently only supported for surface deformables). @@ -93,14 +114,20 @@ For a full description of all available properties, refer to the `PhysX deformab Material Changes ^^^^^^^^^^^^^^^^ -The old :class:`DeformableBodyMaterialCfg` (from ``isaaclab.sim``) has been replaced by a new hierarchy in -``isaaclab_physx``: +The deformable material hierarchy is now split by backend: -- :class:`~isaaclab_physx.sim.DeformableBodyMaterialCfg` — for volume deformables. Contains ``density``, +- :class:`~isaaclab.sim.DeformableBodyMaterialBaseCfg` and + :class:`~isaaclab.sim.SurfaceDeformableBodyMaterialBaseCfg` — empty base classes for backend-specific + deformable material configs. +- :class:`~isaaclab_physx.sim.PhysxDeformableBodyMaterialCfg` — for PhysX volume deformables. Contains ``density``, ``static_friction``, ``dynamic_friction``, ``youngs_modulus``, ``poissons_ratio``, and ``elasticity_damping``. -- :class:`~isaaclab_physx.sim.SurfaceDeformableBodyMaterialCfg` — extends the volume material config with +- :class:`~isaaclab_physx.sim.PhysxSurfaceDeformableBodyMaterialCfg` — extends the PhysX volume material config with surface-specific properties: ``surface_thickness``, ``surface_stretch_stiffness``, ``surface_shear_stiffness``, ``surface_bend_stiffness``, and ``bend_damping``. +- :class:`~isaaclab_newton.sim.spawners.materials.NewtonDeformableBodyMaterialCfg` and + :class:`~isaaclab_newton.sim.spawners.materials.NewtonSurfaceDeformableBodyMaterialCfg` contain Newton-specific + fields such as density, particle radius, direct Lame parameters ``k_mu``/``k_lambda`` for volume deformables, + and VBD stiffness parameters for surface deformables. The old ``damping_scale`` property has been removed. Use ``elasticity_damping`` directly instead. @@ -139,19 +166,19 @@ Volume Deformable (Before and After) **After**: .. code-block:: python - :emphasize-lines: 1,2,3 + :emphasize-lines: 1,2 import isaaclab.sim as sim_utils - from isaaclab_physx.assets import DeformableObject, DeformableObjectCfg - from isaaclab_physx.sim import DeformableBodyPropertiesCfg, DeformableBodyMaterialCfg + from isaaclab.assets import DeformableObject, DeformableObjectCfg + from isaaclab_physx.sim import PhysxDeformableBodyMaterialCfg, PhysxDeformableBodyPropertiesCfg cfg = DeformableObjectCfg( prim_path="/World/Origin.*/Cube", spawn=sim_utils.MeshCuboidCfg( size=(0.2, 0.2, 0.2), - deformable_props=DeformableBodyPropertiesCfg(), + deformable_props=PhysxDeformableBodyPropertiesCfg(), visual_material=sim_utils.PreviewSurfaceCfg(), - physics_material=DeformableBodyMaterialCfg(poissons_ratio=0.4, youngs_modulus=1e5), + physics_material=PhysxDeformableBodyMaterialCfg(poissons_ratio=0.4, youngs_modulus=1e5), ), ) cube_object = DeformableObject(cfg=cfg) @@ -159,23 +186,23 @@ Volume Deformable (Before and After) Surface Deformable (New) ^^^^^^^^^^^^^^^^^^^^^^^^ -Surface deformables use :class:`~isaaclab.sim.spawners.meshes.MeshSquareCfg` for 2D meshes, combined with -:class:`~isaaclab_physx.sim.SurfaceDeformableBodyMaterialCfg`: +Surface deformables use :class:`~isaaclab.sim.spawners.meshes.MeshRectangleCfg` for 2D meshes, combined with +:class:`~isaaclab_physx.sim.PhysxSurfaceDeformableBodyMaterialCfg`: .. code-block:: python import isaaclab.sim as sim_utils - from isaaclab_physx.assets import DeformableObject, DeformableObjectCfg - from isaaclab_physx.sim import DeformableBodyPropertiesCfg, SurfaceDeformableBodyMaterialCfg + from isaaclab.assets import DeformableObject, DeformableObjectCfg + from isaaclab_physx.sim import PhysxDeformableBodyPropertiesCfg, PhysxSurfaceDeformableBodyMaterialCfg cfg = DeformableObjectCfg( prim_path="/World/Origin.*/Cloth", - spawn=sim_utils.MeshSquareCfg( - size=1.5, + spawn=sim_utils.MeshRectangleCfg( + size=(1.5, 1.5), resolution=(21, 21), - deformable_props=DeformableBodyPropertiesCfg(), + deformable_props=PhysxDeformableBodyPropertiesCfg(), visual_material=sim_utils.PreviewSurfaceCfg(), - physics_material=SurfaceDeformableBodyMaterialCfg(poissons_ratio=0.4, youngs_modulus=1e5), + physics_material=PhysxSurfaceDeformableBodyMaterialCfg(poissons_ratio=0.4, youngs_modulus=1e5), ), ) cloth_object = DeformableObject(cfg=cfg) @@ -189,8 +216,8 @@ Deformable properties can also be applied to imported USD assets using .. code-block:: python import isaaclab.sim as sim_utils - from isaaclab_physx.assets import DeformableObject, DeformableObjectCfg - from isaaclab_physx.sim import DeformableBodyPropertiesCfg, DeformableBodyMaterialCfg + from isaaclab.assets import DeformableObject, DeformableObjectCfg + from isaaclab_physx.sim import PhysxDeformableBodyMaterialCfg, PhysxDeformableBodyPropertiesCfg from isaaclab.utils.assets import ISAACLAB_NUCLEUS_DIR @@ -198,8 +225,8 @@ Deformable properties can also be applied to imported USD assets using prim_path="/World/Origin.*/Teddy", spawn=sim_utils.UsdFileCfg( usd_path=f"{ISAACLAB_NUCLEUS_DIR}/Objects/Teddy_Bear/teddy_bear.usd", - deformable_props=DeformableBodyPropertiesCfg(), - physics_material=DeformableBodyMaterialCfg(poissons_ratio=0.4, youngs_modulus=1e5), + deformable_props=PhysxDeformableBodyPropertiesCfg(), + physics_material=PhysxDeformableBodyMaterialCfg(poissons_ratio=0.4, youngs_modulus=1e5), scale=[0.05, 0.05, 0.05], ), ) @@ -210,12 +237,13 @@ Limitations ~~~~~~~~~~~ - **Kinematic targets are volume-only.** Calling - :meth:`~isaaclab_physx.assets.DeformableObject.write_nodal_kinematic_target_to_sim_index` on a surface + :meth:`~isaaclab.assets.DeformableObject.write_nodal_kinematic_target_to_sim_index` on a surface deformable will raise a ``ValueError``. - **Surface-specific solver properties** (``collision_pair_update_frequency``, ``collision_iteration_multiplier``) have no effect on volume deformables. -- **Deformables are PhysX-only.** The ``isaaclab_physx`` extension is required; other physics backends - do not support deformable bodies through Isaac Lab yet. +- **Newton deformables are experimental.** They are implemented in + :mod:`isaaclab_contrib.deformable` and currently target VBD-based solvers and + coupled rigid-deformable workflows. .. _Omni Physics documentation: https://docs.omniverse.nvidia.com/kit/docs/omni_physics/110.0/dev_guide/deformables/deformable_bodies.html diff --git a/docs/source/migration/migrating_to_isaaclab_3-0.rst b/docs/source/migration/migrating_to_isaaclab_3-0.rst index efa7d867802..fc8657658aa 100644 --- a/docs/source/migration/migrating_to_isaaclab_3-0.rst +++ b/docs/source/migration/migrating_to_isaaclab_3-0.rst @@ -69,9 +69,9 @@ New ``isaaclab_physx`` and ``isaaclab_newton`` Extensions Two new backend extensions have been introduced: -- **``isaaclab_physx``** — PhysX-specific implementations of all asset and sensor classes. -- **``isaaclab_newton``** — Newton-specific implementations of asset classes (Articulation and - RigidObject). +- **``isaaclab_physx``** — PhysX-specific implementations of asset and sensor classes. +- **``isaaclab_newton``** — Newton-specific implementations of supported asset classes, including + articulations, rigid objects, and deformable objects. The following classes have been moved to ``isaaclab_physx``: @@ -81,17 +81,18 @@ The following classes have been moved to ``isaaclab_physx``: * - Isaac Lab 2.x - Isaac Lab 3.0 - * - ``from isaaclab.assets import DeformableObject`` - - ``from isaaclab_physx.assets import DeformableObject`` - * - ``from isaaclab.assets import DeformableObjectCfg`` - - ``from isaaclab_physx.assets import DeformableObjectCfg`` - * - ``from isaaclab.assets import DeformableObjectData`` - - ``from isaaclab_physx.assets import DeformableObjectData`` * - ``from isaaclab.assets import SurfaceGripper`` - ``from isaaclab_physx.assets import SurfaceGripper`` * - ``from isaaclab.assets import SurfaceGripperCfg`` - ``from isaaclab_physx.assets import SurfaceGripperCfg`` +.. note:: + + Deformable object public APIs remain in the backend-neutral ``isaaclab`` + package. Continue importing :class:`~isaaclab.assets.DeformableObject`, + :class:`~isaaclab.assets.DeformableObjectCfg`, and + :class:`~isaaclab.assets.DeformableObjectData` from ``isaaclab.assets``. + .. note:: The ``isaaclab_physx`` extension is installed automatically with Isaac Lab. No additional @@ -382,13 +383,17 @@ release. The old soft body API has been deprecated and replaced by two distinct types: **volume deformables** (3D FEM tetrahedral meshes) and **surface deformables** (2D triangle cloth meshes). The deformable type is determined by the physics material assigned: -- :class:`~isaaclab_physx.sim.DeformableBodyMaterialCfg` for volume deformables. -- :class:`~isaaclab_physx.sim.SurfaceDeformableBodyMaterialCfg` for surface deformables. +- :class:`~isaaclab_physx.sim.PhysxDeformableBodyMaterialCfg` for PhysX volume deformables. +- :class:`~isaaclab_physx.sim.PhysxSurfaceDeformableBodyMaterialCfg` for PhysX surface deformables. +- :class:`~isaaclab_newton.sim.spawners.materials.NewtonDeformableBodyMaterialCfg` for Newton volume deformables. +- :class:`~isaaclab_newton.sim.spawners.materials.NewtonSurfaceDeformableBodyMaterialCfg` for Newton surface + deformables. -All deformable-related classes have moved from ``isaaclab`` to ``isaaclab_physx``, as shown -in the import table above. Several properties on -:class:`~isaaclab_physx.sim.DeformableBodyPropertiesCfg` have been removed or added to match -the new Omni Physics schema. +Deformable property and material cfgs are backend-specific. Several properties on +:class:`~isaaclab_physx.sim.PhysxDeformableBodyPropertiesCfg` have been removed or added to +match the new Omni Physics schema. The common +:class:`~isaaclab.sim.DeformableBodyPropertiesBaseCfg` is now empty; OmniPhysics +deformable body fields are owned by :class:`~isaaclab_physx.sim.PhysxDeformableBodyPropertiesCfg`. For a comprehensive guide covering the full deformable API migration — including removed and added properties, material changes, code examples for both volume and surface deformables, and @@ -877,7 +882,7 @@ Here's a complete example showing how to update your code: .. code-block:: python - from isaaclab_physx.assets import DeformableObject, DeformableObjectCfg + from isaaclab.assets import DeformableObject, DeformableObjectCfg from isaaclab_physx.assets import SurfaceGripper, SurfaceGripperCfg from isaaclab.assets import RigidObjectCollection # unchanged @@ -1219,7 +1224,7 @@ All ``.data.*`` properties on asset and sensor classes now return the underlying ``wp.array`` and exposes explicit ``.torch`` and ``.warp`` accessors. This change applies to all asset classes (:class:`~isaaclab.assets.Articulation`, :class:`~isaaclab.assets.RigidObject`, :class:`~isaaclab.assets.RigidObjectCollection`, -:class:`~isaaclab_physx.assets.DeformableObject`) and all sensor classes +:class:`~isaaclab.assets.DeformableObject`) and all sensor classes (:class:`~isaaclab_physx.sensors.ContactSensor`, :class:`~isaaclab_physx.sensors.Imu`, :class:`~isaaclab_physx.sensors.Pva`, :class:`~isaaclab_physx.sensors.FrameTransformer`). @@ -1276,8 +1281,8 @@ Common patterns that need updating: - ``isaaclab`` / ``isaaclab_physx`` * - :class:`~isaaclab.assets.RigidObjectCollection` - ``isaaclab`` / ``isaaclab_physx`` - * - :class:`~isaaclab_physx.assets.DeformableObject` - - ``isaaclab_physx`` + * - :class:`~isaaclab.assets.DeformableObject` + - ``isaaclab`` / ``isaaclab_physx`` / ``isaaclab_newton`` * - :class:`~isaaclab_physx.sensors.ContactSensor` - ``isaaclab_physx`` * - :class:`~isaaclab_physx.sensors.Imu` diff --git a/docs/source/overview/core-concepts/multi_backend_architecture.rst b/docs/source/overview/core-concepts/multi_backend_architecture.rst index 5d894c6820f..55c524b22c3 100644 --- a/docs/source/overview/core-concepts/multi_backend_architecture.rst +++ b/docs/source/overview/core-concepts/multi_backend_architecture.rst @@ -43,6 +43,10 @@ This pattern applies to all simulation components: - :class:`~isaaclab.assets.RigidObject` - :class:`~isaaclab_physx.assets.RigidObject` - :class:`~isaaclab_newton.assets.RigidObject` + * - Deformable Object + - :class:`~isaaclab.assets.DeformableObject` + - :class:`~isaaclab_physx.assets.DeformableObject` + - :class:`~isaaclab_newton.assets.DeformableObject` * - Contact Sensor - :class:`~isaaclab.sensors.ContactSensor` - :class:`~isaaclab_physx.sensors.ContactSensor` @@ -52,7 +56,7 @@ This pattern applies to all simulation components: - :class:`~isaaclab_physx.renderers.IsaacRtxRenderer` - :class:`~isaaclab_newton.renderers.NewtonWarpRenderer` * - Scene Data Backend - - :class:`~isaaclab.physics.SceneDataBackend` + - :class:`~isaaclab.scene_data.SceneDataBackend` - ``PhysxSceneDataBackend`` (in :mod:`isaaclab_physx.physics`) - ``NewtonSceneDataBackend`` (in :mod:`isaaclab_newton.physics`) * - Cloner @@ -254,6 +258,8 @@ the established conventions: │ │ └── articulation_data.py │ ├── rigid_object/ │ │ └── ... + │ ├── deformable_object/ + │ │ └── ... │ └── rigid_object_collection/ │ └── ... ├── sensors/ @@ -266,14 +272,15 @@ the established conventions: **2. Implement the physics manager:** -The manager must expose a :class:`~isaaclab.physics.SceneDataBackend` so that -:class:`~isaaclab.scene.scene_data_provider.SceneDataProvider` can read your backend's body +The manager must expose a :class:`~isaaclab.scene_data.SceneDataBackend` so that +:class:`~isaaclab.scene_data.SceneDataProvider` can read your backend's body transforms in a Warp-native format that renderers and visualizers consume directly. .. code-block:: python # isaaclab_mybackend/physics/mybackend_manager.py - from isaaclab.physics import PhysicsManager, SceneDataBackend, SceneDataFormat + from isaaclab.physics import PhysicsManager + from isaaclab.scene_data import SceneDataBackend, SceneDataFormat class MyBackendSceneDataBackend(SceneDataBackend): @@ -384,4 +391,6 @@ See Also - :doc:`/source/features/hydra` — preset system for multi-backend environment configurations - :doc:`physical-backends/index` — feature matrix and per-backend guides (PhysX, Newton, OvPhysX) - :doc:`physical-backends/newton/index` — Newton backend guide +- :doc:`physical-backends/newton/newton-manager-abstraction` — adding Newton solver managers and + coupled solvers - :doc:`renderers` — renderer backend architecture diff --git a/docs/source/overview/core-concepts/physical-backends/newton/index.rst b/docs/source/overview/core-concepts/physical-backends/newton/index.rst index 087adda9cb2..71e1ffed14e 100644 --- a/docs/source/overview/core-concepts/physical-backends/newton/index.rst +++ b/docs/source/overview/core-concepts/physical-backends/newton/index.rst @@ -20,10 +20,18 @@ have successfully deployed a Newton-trained locomotion policy to a G1 robot. Newton can support `multiple solvers `_ for handling different types of physics simulation. The Isaac Lab integration ships -two solver pages: +the following solver pages: * :doc:`mjwarp-solver` — the primary, validated solver path. * :doc:`kamino-solver` — beta support on selected classic tasks. +* :doc:`using-vbd-solver` — experimental VBD solver for cloth and soft bodies, + available through :mod:`isaaclab_contrib.deformable` and the MJWarp + VBD or + Featherstone + VBD coupled managers. + +Each solver is exposed as a small subclass of +:class:`~isaaclab_newton.physics.NewtonManager`. See +:doc:`newton-manager-abstraction` for the developer-facing guide to adding a +new solver or a coupled solver. During the beta phase, breaking changes and incomplete documentation are still expected. Official support and debugging assistance will follow once the framework @@ -41,5 +49,7 @@ new backend, see :doc:`../../multi_backend_architecture`. supported-features mjwarp-solver kamino-solver + using-vbd-solver + newton-manager-abstraction warp-environments warp-env-migration diff --git a/docs/source/overview/core-concepts/physical-backends/newton/mjwarp-solver.rst b/docs/source/overview/core-concepts/physical-backends/newton/mjwarp-solver.rst index 805f51a7ec2..18bdeabfc61 100644 --- a/docs/source/overview/core-concepts/physical-backends/newton/mjwarp-solver.rst +++ b/docs/source/overview/core-concepts/physical-backends/newton/mjwarp-solver.rst @@ -7,7 +7,9 @@ for the Newton backend in Isaac Lab. It is enabled by setting :class:`~isaaclab_newton.physics.MJWarpSolverCfg`, usually exposed as the ``newton_mjwarp`` physics preset on a task configuration. Newton ships with beta support for an alternative Kamino solver — see :doc:`kamino-solver` and -:ref:`hydra-backend-solver-presets` for how presets are selected. +:ref:`hydra-backend-solver-presets` for how presets are selected. For details +on how solver-specific managers are implemented, or how to add a new solver +manager, see :doc:`newton-manager-abstraction`. .. note:: diff --git a/docs/source/overview/core-concepts/physical-backends/newton/newton-manager-abstraction.rst b/docs/source/overview/core-concepts/physical-backends/newton/newton-manager-abstraction.rst new file mode 100644 index 00000000000..312d0c5bb79 --- /dev/null +++ b/docs/source/overview/core-concepts/physical-backends/newton/newton-manager-abstraction.rst @@ -0,0 +1,237 @@ +Newton Manager Abstraction +========================== + +Newton exposes multiple solver families, and Isaac Lab keeps that flexibility by +making each solver an implementation detail of a small +:class:`~isaaclab_newton.physics.NewtonManager` subclass. The simulation context +still sees a normal physics manager; the solver configuration decides which +manager class is used. + +For most new Newton solvers, the integration surface is intentionally small: + +* define a solver config that inherits from + :class:`~isaaclab_newton.physics.NewtonSolverCfg`; +* point the config's ``class_type`` at a manager subclass; +* implement ``_build_solver()`` in that manager; +* set the three base-manager slots: ``_solver``, ``_use_single_state``, and + ``_needs_collision_pipeline``. + +The existing MuJoCo Warp, XPBD, Featherstone, and Kamino managers are examples +of this pattern. + + +Adding a Solver Manager +----------------------- + +The solver config carries both user-tunable solver parameters and the manager +dispatch target: + +.. code-block:: python + + from isaaclab_newton.physics import NewtonManager, NewtonSolverCfg + from isaaclab.utils.configclass import configclass + + + @configclass + class MySolverCfg(NewtonSolverCfg): + class_type: type[NewtonManager] | str = "{DIR}.my_solver_manager:NewtonMySolverManager" + solver_type: str = "my_solver" + iterations: int = 16 + + +``NewtonCfg`` copies ``solver_cfg.class_type`` into its own ``class_type`` in +``__post_init__``. User code keeps the normal shape: + +.. code-block:: python + + from isaaclab.sim import SimulationCfg + from isaaclab_newton.physics import NewtonCfg + + sim_cfg = SimulationCfg( + physics=NewtonCfg( + solver_cfg=MySolverCfg(iterations=32), + num_substeps=2, + ) + ) + + +The manager then owns solver construction: + +.. code-block:: python + + from newton import Model + from newton.solvers import SolverMySolver + + from isaaclab_newton.physics import NewtonManager + + + class NewtonMySolverManager(NewtonManager): + @classmethod + def _build_solver(cls, model: Model, solver_cfg: MySolverCfg) -> None: + NewtonManager._solver = SolverMySolver(model, iterations=solver_cfg.iterations) + NewtonManager._use_single_state = False + NewtonManager._needs_collision_pipeline = True + + +``_use_single_state`` tells the base manager whether the solver advances in +place or swaps input/output states. ``_needs_collision_pipeline`` tells the base +manager whether to allocate and pass Newton collision-pipeline contacts to the +solver. A solver with its own internal contact detector can set it to ``False``. + +Optional Overrides +------------------ + +Most managers only implement ``_build_solver()``. Override more only when the +solver actually needs it: + +* ``_initialize_contacts()``: allocate custom contact buffers or support an + internal contact detector. +* ``_step_solver(state_0, state_1, control, substep_dt)``: change one substep of + solver execution while keeping the base simulation loop. +* ``_simulate_physics_only()``: add per-step work around the base substep loop, + such as rebuilding a BVH. +* ``step()``: handle solver-specific reset masks, graph capture, or model-change + notification before delegating to the base manager. +* ``start_simulation()`` or ``instantiate_builder_from_stage()``: customize model + building or post-finalize setup. +* ``_solver_specific_clear()``: release any class-level state owned by the + solver manager. + +Keep the manager name prefixed with ``Newton`` and the solver config grouped +with the other Newton solver configs so autocomplete and backend discovery stay +predictable. + + +Custom Coupled Solvers +---------------------- + +Coupled solvers use the same abstraction. Instead of wrapping one Newton solver, +a coupled manager constructs two or more sub-solvers and overrides +``_step_solver()`` to define the substep order. +That means a custom coupling usually needs only a config that stores existing +solver configs plus a manager that defines how data flows between them; the +component solvers can stay unchanged. + +The MJWarp + VBD deformable manager is a concrete example: + +* :class:`~isaaclab_contrib.deformable.CoupledMJWarpVBDSolverCfg` stores a + ``rigid_solver_cfg`` for :class:`~isaaclab_newton.physics.MJWarpSolverCfg`, a + ``soft_solver_cfg`` for :class:`~isaaclab_contrib.deformable.VBDSolverCfg`, + and a ``coupling_mode``. +* ``NewtonCoupledMJWarpVBDManager._build_solver()`` constructs + ``SolverMuJoCo`` and ``SolverVBD`` from those sub-configs. +* ``_step_solver()`` dispatches to either one-way or two-way coupling. +* The base ``NewtonManager`` still owns state allocation, substep iteration, + Fabric synchronization, and reset/clear lifecycle. + +The two-way MJWarp + VBD substep stays compact because it is expressed as a +short coupling algorithm: + +.. admonition:: Algorithm: Two-Way MJWarp + VBD Substep + :class: note + + **Inputs:** rigid body state, deformable particle state, and the shared + Newton collision pipeline. + + **Output:** updated rigid body and deformable particle state for one Newton + substep. + + 1. **Reset force accumulators.** + Clear the rigid body and particle force buffers before evaluating the + next contact pass. + + 2. **Detect coupled contacts.** + Run Newton collision detection once over the current rigid and + deformable state. + + 3. **Apply soft-to-rigid reactions.** + Inject body-particle contact reactions into ``body_f`` so the rigid + bodies can be pushed back by the deformable contact penalties. + + 4. **Advance the rigid solver.** + Step the MJWarp rigid solver with the coupled contact forces applied. + + 5. **Preserve shared contacts for the soft solve.** + Clear particle forces written during the rigid step while keeping the + detected contact information available. + + 6. **Advance the deformable solver.** + Step the VBD soft solver against the same coupled contacts. + + +This keeps the custom part focused on the coupling policy. The manager does not +need to reimplement scene loading, asset buffers, reset handling, or the outer +simulation loop. + +.. figure:: ../../../../_static/newton/franka-mjwarp-vbd-coupling.png + :align: center + :figwidth: 480px + :class: square-crop-figure + :alt: Franka manipulating a deformable object with MJWarp and VBD coupling + + Franka manipulation using MJWarp for rigid bodies and VBD for the deformable + object. + +You can exercise this coupling path with the Franka soft-body lifting task: + +.. code-block:: bash + + ./isaaclab.sh -p scripts/environments/zero_agent.py --task Isaac-Lift-Soft-Franka-v0 --num_envs 1 --visualizer kit + +For the surface-deformable cloth variant, use ``--task Isaac-Lift-Cloth-Franka-v0``. + + +This environment configures +:class:`~isaaclab_contrib.deformable.CoupledMJWarpVBDSolverCfg` with +``coupling_mode="two_way"``. + +Tuning the Franka Soft-Body Lift +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Tune the coupled contact behavior before training a policy: + +* Start with ``coupling_mode="two_way"``. Compared with one-way coupling, two-way + coupling can prevent clipping more easily because body-particle contact + penalties can push the robot back instead of only moving the deformable. +* Use a small scripted grasp/lift check before training to confirm that grasping + is possible and to inspect what clips when the grasp fails. +* Lower the arm actuator stiffness enough that the arm can respond to contact + penalties. Prefer the arm being pushed back over the gripper clipping into the + deformable. +* Tune :attr:`~isaaclab_contrib.deformable.NewtonModelCfg.soft_contact_ke` + first. Increase it only as much as needed to prevent clipping, then adjust + :attr:`~isaaclab_contrib.deformable.NewtonModelCfg.soft_contact_mu` so the + gripper can carry the object without requiring an obviously unphysical + friction value. Use + :attr:`~isaaclab_contrib.deformable.NewtonModelCfg.soft_contact_kd` for + stabilization if contacts chatter. +* Tune the ``soft_contact_*`` values together with ``shape_material_*`` values + because rigid shape material parameters also affect the effective contact. +* If ``soft_contact_ke`` is not sufficient, or ``soft_contact_mu`` must be + unphysically high, tune the Franka arm and hand actuator stiffness and maximum + effort. For the gripper command, fully close the fingers and let the actuator + maximum effort limit the actual squeeze. +* If the deformable no longer visibly deforms, ``soft_contact_ke`` is likely too + high. +* If contacts are unstable or missed, increase the deformable mesh resolution or + increase ``particle_radius`` in the deformable material so contact is detected + earlier from a larger distance. +* If the rigid shapes still clip through the deformable, increase + :attr:`~isaaclab_contrib.deformable.VBDSolverCfg.iterations`; more VBD + iterations can improve contact convergence. + + +When to Add a Coupled Manager +----------------------------- + +Add a coupled manager when one solver cannot own the whole model step by itself: + +* rigid bodies should use one solver while particles or cloth use another; +* contact detection is shared, but each solver consumes the contacts + differently; +* you need a custom force, impulse, or state exchange between solvers; +* the substep order is part of the algorithm. + +Use a normal single-solver manager when all physics can be advanced by one +Newton solver. Use a coupled manager only for the small amount of glue that is +truly solver-specific. diff --git a/docs/source/overview/core-concepts/physical-backends/newton/supported-features.rst b/docs/source/overview/core-concepts/physical-backends/newton/supported-features.rst index d7fd6aabf0d..def15735fc9 100644 --- a/docs/source/overview/core-concepts/physical-backends/newton/supported-features.rst +++ b/docs/source/overview/core-concepts/physical-backends/newton/supported-features.rst @@ -35,10 +35,18 @@ isaaclab * Rigid Object and Rigid Object Collection APIs * Sensors: Contact Sensor, IMU, Frame Transformer, Joint Wrench, PVA * Direct and Manager-based single-agent workflows +* Backend-neutral deformable object API * Omniverse Kit visualizer (when Isaac Sim is installed) * Newton-Warp visualizer (kit-less) * Tiled rendering via the Newton-Warp renderer +isaaclab_contrib +^^^^^^^^^^^^^^^^ + +* Experimental Newton deformable objects +* VBD deformable solver (see :doc:`using-vbd-solver`) +* Coupled MJWarp + VBD and Featherstone + VBD solver managers + The following sensors are backend-agnostic (implemented in ``isaaclab`` core) and work transparently with Newton: @@ -71,6 +79,8 @@ Manager-based workflows: Unitree G1, Go1, Go2, Unitree H1, Spot * Locomotion velocity, rough terrain: Anymal-C, Cassie, Go1, Go2 * Manipulation: reach (Franka, UR10), cabinet, dexsuite +* Manipulation lift with deformable objects: Franka soft-body lift, Franka cloth + lift (via coupled MJWarp + VBD) Solver Coverage @@ -80,15 +90,19 @@ Solver Coverage * **Kamino solver**: beta. Currently validated on ``Isaac-Cartpole-Direct-v0``, ``Isaac-Ant-Direct-v0``, ``Isaac-Cartpole-v0``, and ``Isaac-Ant-v0``. See :doc:`kamino-solver`. - -Other Newton solvers (e.g. VBD) are not yet exposed through Isaac Lab. +* **VBD solver**: experimental, exposed through :mod:`isaaclab_contrib.deformable` + for cloth and soft-body simulation. Most often used inside the coupled + MJWarp + VBD or Featherstone + VBD managers so one solver advances rigid + bodies and VBD advances deformable particles. See :doc:`using-vbd-solver` + and :doc:`newton-manager-abstraction`. Known Gaps ---------- -* Soft bodies, particles, and other non-rigid PhysX features are not yet - available through Newton. +* Soft bodies and particles are available through the experimental VBD path in + :mod:`isaaclab_contrib.deformable`; other non-rigid PhysX features are not + yet covered. * Behaviour on stiff contact stacks can diverge from PhysX; expect to retune contact and substep parameters when porting tasks across backends. * Multi-agent and self-play workflows are not yet wired up for Newton. diff --git a/docs/source/overview/core-concepts/physical-backends/newton/using-vbd-solver.rst b/docs/source/overview/core-concepts/physical-backends/newton/using-vbd-solver.rst new file mode 100644 index 00000000000..4e50b1f4339 --- /dev/null +++ b/docs/source/overview/core-concepts/physical-backends/newton/using-vbd-solver.rst @@ -0,0 +1,344 @@ +.. _newton-using-vbd: + +Using the VBD Solver +==================== + +Vertex Block Descent (VBD) is a Newton solver for cloth and soft-body +simulation. In Isaac Lab, VBD is enabled by selecting a +:class:`~isaaclab_newton.physics.NewtonCfg` whose ``solver_cfg`` is provided by +:mod:`isaaclab_contrib.deformable`. + +VBD support is experimental. The solver managers, configuration fields, and +recommended tuning values may change while Newton deformable support is under +active development. A task that works with PhysX or with Newton's MuJoCo-Warp +solver may still need deformable assets, materials, contacts, and coupling tuned +before it works well with VBD. + +VBD is usually exposed through a task-specific physics preset rather than a +general ``newton_vbd`` preset. Deformable-only scenes can use +:class:`~isaaclab_contrib.deformable.VBDSolverCfg` directly. Robot or +rigid-body scenes usually use +:class:`~isaaclab_contrib.deformable.CoupledMJWarpVBDSolverCfg` or +:class:`~isaaclab_contrib.deformable.CoupledFeatherstoneVBDSolverCfg` so one +solver advances rigid bodies and VBD advances deformable particles. + +Start from a Supported Deformable Task +-------------------------------------- + +Before adding VBD to a new task, first run one of the experimental Franka +deformable tasks: + +.. code-block:: bash + + ./isaaclab.sh -p scripts/environments/zero_agent.py --task Isaac-Lift-Soft-Franka-v0 --num_envs 1 --visualizer kit + +For the surface-deformable cloth variant, use: + +.. code-block:: bash + + ./isaaclab.sh -p scripts/environments/zero_agent.py --task Isaac-Lift-Cloth-Franka-v0 --num_envs 1 --visualizer kit + +Both tasks configure MJWarp for the rigid Franka and VBD for the deformable +object through +:class:`~isaaclab_contrib.deformable.CoupledMJWarpVBDSolverCfg`. +Use these tasks as starting points for asset setup, solver coupling, and contact +tuning. + +Add a VBD Physics Preset +------------------------ + +Tasks that support multiple physics options usually store ``SimulationCfg.physics`` +as a :class:`~isaaclab_tasks.utils.hydra.PresetCfg`. For deformable Newton tasks, +the preset can use a small :class:`~isaaclab_newton.physics.NewtonCfg` subclass +to carry :class:`~isaaclab_contrib.deformable.NewtonModelCfg` alongside the +normal Newton fields: + +.. code-block:: python + + from isaaclab.utils.configclass import configclass + from isaaclab_newton.physics import NewtonCfg + + from isaaclab_contrib.deformable import NewtonModelCfg + + + @configclass + class DeformableNewtonCfg(NewtonCfg): + model_cfg: NewtonModelCfg | None = None + + +The Franka soft-body task defines a ``newton_mjwarp_vbd`` preset that couples +MJWarp and VBD: + +.. literalinclude:: ../../../../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/lift_franka_soft/franka_soft_env_cfg.py + :language: python + :start-at: class PhysicsCfg + :end-at: default = newton_mjwarp_vbd + :emphasize-lines: 4-32 + +The important pieces are: + +* Add a Newton physics preset whose value is ``DeformableNewtonCfg``. +* Use :class:`~isaaclab_contrib.deformable.CoupledMJWarpVBDSolverCfg` when rigid + bodies and deformables must interact in the same scene. +* Use ``soft_solver_cfg=VBDSolverCfg(integrate_with_external_rigid_solver=True)`` + inside a coupled solver so VBD advances only the deformable particles. +* Add :class:`~isaaclab_contrib.deformable.NewtonModelCfg` when body-particle or + self-contact values need task-level tuning. +* Keep the preset at the same config path used by the task's + :class:`~isaaclab.sim.SimulationCfg`, for example ``env.sim.physics``. + +You can select the deformable Newton preset globally: + +.. code-block:: bash + + ./isaaclab.sh -p scripts/reinforcement_learning/rsl_rl/train.py --task=Isaac-Lift-Soft-Franka-v0 presets=newton_mjwarp_vbd + +or select the physics field directly: + +.. code-block:: bash + + ./isaaclab.sh -p scripts/reinforcement_learning/rsl_rl/train.py --task=Isaac-Lift-Soft-Franka-v0 env.sim.physics=newton_mjwarp_vbd + +Use the direct path override when only one task field should use the VBD preset. +Use ``presets=newton_mjwarp_vbd`` when you want every matching preset field in +the task config to resolve to that preset. Isaac Lab training scripts accept +these Hydra overrides after the regular command line flags; no separator is +needed for the examples above. + + +Check Task and Asset Compatibility +---------------------------------- + +VBD uses the Newton model built from the task assets. When adding VBD to a new +task, validate the following before tuning solver parameters: + +* The task must already be compatible with the Newton backend. If a rigid-only + ``newton_mjwarp`` preset fails during model construction, fix the asset or task + configuration first. +* The scene must include Newton-compatible deformable assets and materials. Use + :class:`~isaaclab_newton.sim.spawners.materials.NewtonDeformableBodyMaterialCfg` + for volume deformables and + :class:`~isaaclab_newton.sim.spawners.materials.NewtonSurfaceDeformableBodyMaterialCfg` + for cloth or surface deformables. +* Coupled robot tasks should start with ``coupling_mode="two_way"`` when the + robot should feel contact forces from the deformable object. +* Contact-heavy scenes usually need task-specific ``num_substeps``, + :class:`~isaaclab_contrib.deformable.VBDSolverCfg`, and + :class:`~isaaclab_contrib.deformable.NewtonModelCfg` values. Start from the + Franka soft-body or cloth preset that most closely resembles the scene. +* Use a small visual smoke test before training. Confirm that the deformable + spawns, renders, deforms, and contacts rigid bodies as expected. + +VBD Solver Parameters +--------------------- + +The following fields are specific to +:class:`~isaaclab_contrib.deformable.VBDSolverCfg`. They are grouped by the part +of the solver they affect. + +Core Solve +^^^^^^^^^^ + +.. list-table:: + :header-rows: 1 + :widths: 30 70 + + * - Parameter + - Description + * - ``iterations`` + - Default: ``10``. Number of VBD iterations per substep. Increasing this value improves deformation and contact convergence, especially for stiff materials or rigid gripper contacts, but increases runtime. + * - ``integrate_with_external_rigid_solver`` + - Default: ``False``. Set to ``True`` when VBD is used inside a coupled solver so the rigid sub-solver owns rigid-body integration. Leave ``False`` for deformable-only VBD scenes. + + +Self-Contact +^^^^^^^^^^^^ + +.. list-table:: + :header-rows: 1 + :widths: 30 70 + + * - Parameter + - Description + * - ``particle_enable_self_contact`` + - Default: ``False``. Enables deformable self-contact. Use this for cloth folds or soft bodies that collide with themselves. It increases contact work and usually needs additional tuning. + * - ``particle_self_contact_radius`` + - Default: ``0.005`` [m]. Effective self-contact thickness. VBD applies vertex-triangle and edge-edge self-contact response when the current primitive distance is smaller than this radius. + * - ``particle_self_contact_margin`` + - Default: ``0.005`` [m]. Self-contact candidate search distance. VBD uses this envelope when building self-contact lists, then applies contact response using ``particle_self_contact_radius``. Keep this greater than or equal to the radius to avoid missed contacts. + * - ``particle_collision_detection_interval`` + - Default: ``-1``. Controls how often self-contact detection runs. A negative value detects before initialization only. ``0`` detects before and immediately after initialization. A positive value ``k`` detects before every ``k`` VBD iterations. + * - ``particle_vertex_contact_buffer_size`` + - Default: ``32``. Preallocation size for each vertex's vertex-triangle self-contact buffer. Increase it if dense folds or high-resolution cloth exceed the default capacity. + * - ``particle_edge_contact_buffer_size`` + - Default: ``64``. Preallocation size for each edge's edge-edge self-contact buffer. Increase it if dense folds or high-resolution cloth exceed the default capacity. + * - ``particle_topological_contact_filter_threshold`` + - Default: ``2``. Filters contacts between mesh primitives that are close in topology. Increase this to suppress contact between neighboring elements of the same surface. Values greater than ``3`` can significantly increase compute time. + * - ``particle_rest_shape_contact_exclusion_radius`` + - Default: ``0.0`` [m]. Filters self-contact candidates whose rest-configuration distance is shorter than this distance. Increase it when rest-neighbor contacts produce unwanted resistance. + + +Coupled Solver Parameters +------------------------- + +Use the coupled solver configs when one solver should advance rigid bodies and +VBD should advance deformables: + +.. list-table:: + :header-rows: 1 + :widths: 30 70 + + * - Parameter + - Description + * - ``rigid_solver_cfg`` + - Rigid-body sub-solver configuration. :class:`~isaaclab_contrib.deformable.CoupledMJWarpVBDSolverCfg` uses :class:`~isaaclab_newton.physics.MJWarpSolverCfg`; :class:`~isaaclab_contrib.deformable.CoupledFeatherstoneVBDSolverCfg` uses :class:`~isaaclab_newton.physics.FeatherstoneSolverCfg`. + * - ``soft_solver_cfg`` + - VBD sub-solver configuration. In coupled scenes, set ``integrate_with_external_rigid_solver=True`` so VBD advances only deformable particles. + * - ``coupling_mode="one_way"`` + - Rigid solver advances first, and VBD reacts to the updated rigid poses. The rigid solver does not feel particle contact forces. + * - ``coupling_mode="two_way"`` + - Contact reactions from deformables are injected into the rigid solver before the rigid step, then VBD advances deformables against the shared contacts. Use this for manipulation tasks where the robot should be pushed back by deformable contact. + * - ``coupling_mode="kinematic"`` + - Available on :class:`~isaaclab_contrib.deformable.CoupledFeatherstoneVBDSolverCfg`. Rigid bodies are kinematically updated by Featherstone, and VBD reacts to them. The rigid solver does not feel particle contacts. + +The rigid solver parameters still matter. For example, MJWarp's ``nconmax`` and +``njmax`` must be large enough for the rigid contacts in the scene, and +``ccd_iterations`` can affect fast rigid contacts near deformables. See +:doc:`mjwarp-solver` for the MJWarp-side parameters. + + +Contact and Material Parameters +------------------------------- + +Contact Model +^^^^^^^^^^^^^ + +:class:`~isaaclab_contrib.deformable.NewtonModelCfg` applies contact parameters +to the finalized Newton model: + +.. list-table:: + :header-rows: 1 + :widths: 30 70 + + * - Parameter + - Description + * - ``soft_contact_ke`` + - Default: ``1.0e3`` [N/m]. Stiffness for body-particle and particle self-contact. Increase it to reduce clipping through rigid shapes or through other deformable particles. If it is too high, the object can stop visibly deforming or require more VBD iterations and substeps. + * - ``soft_contact_kd`` + - Default: ``1.0e-2`` [N*s/m]. Contact damping. Increase it to reduce chatter or bouncing. Too much damping can make contact response sticky or overdamped. + * - ``soft_contact_mu`` + - Default: ``0.5``. Friction coefficient for body-particle and particle self-contact. Increase it when a gripper cannot carry the deformable object without slipping. + * - ``shape_material_ke`` + - Default: ``None`` [N/m]. Optional override for all rigid collision-shape contact stiffness values in the Newton model. Use this when the rigid-side material parsed from the asset is not appropriate for deformable contact. + * - ``shape_material_kd`` + - Default: ``None`` [N*s/m]. Optional override for all rigid collision-shape contact damping values in the Newton model. + * - ``shape_material_mu`` + - Default: ``None``. Optional override for all rigid collision-shape friction values in the Newton model. Body-particle friction depends on both the soft contact and rigid shape friction coefficients. + + +Volume Deformable Materials +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Use +:class:`~isaaclab_newton.sim.spawners.materials.NewtonDeformableBodyMaterialCfg` +for volume deformables: + +.. list-table:: + :header-rows: 1 + :widths: 30 70 + + * - Parameter + - Description + * - ``density`` + - Default: ``1.0`` [kg/m^3]. Material density. Higher density increases particle mass and inertia, so the object accelerates and deforms less for the same contact forces. + * - ``particle_radius`` + - Default: ``0.008`` [m]. Particle contact radius used by Newton. Increase it when contacts are missed or detected too late. If it is too large relative to the mesh resolution, contacts can start too early. + * - ``k_mu`` + - Default: ``1.0e5`` [Pa]. First Lame material parameter. Higher values make the deformable object stiffer and usually require more VBD iterations, more substeps, or a smaller timestep. + * - ``k_lambda`` + - Default: ``1.0e5`` [Pa]. Second Lame material parameter. Higher values make the deformable object stiffer and usually require more VBD iterations, more substeps, or a smaller timestep. + * - ``k_damp`` + - Default: ``0.0`` [Pa*s]. Damping for tetrahedral elements. Increase it to reduce oscillations after deformation, but avoid overdamping if the object should rebound. + + +Surface Deformable Materials +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Use +:class:`~isaaclab_newton.sim.spawners.materials.NewtonSurfaceDeformableBodyMaterialCfg` +for cloth or surface deformables: + +.. list-table:: + :header-rows: 1 + :widths: 30 70 + + * - Parameter + - Description + * - ``density`` + - Default: ``1.0`` [kg/m^3]. Material density. Higher density increases particle mass and inertia. + * - ``particle_radius`` + - Default: ``0.008`` [m]. Particle contact radius used by Newton. + * - ``tri_ke`` + - Default: ``1.0e4`` [Pa]. Triangle area-preserving stiffness. Increase it to reduce cloth stretch. + * - ``tri_ka`` + - Default: ``1.0e4`` [Pa]. Triangle area stiffness. Increase it to reduce cloth area change. + * - ``tri_kd`` + - Default: ``1.5e-6`` [Pa*s]. Triangle area damping. Increase it to reduce cloth vibration after stretching. + * - ``edge_ke`` + - Default: ``5.0`` [N*m]. Bending stiffness. Increase it for stiffer cloth folds; decrease it for softer draping. + * - ``edge_kd`` + - Default: ``1.0e-2`` [N*m*s]. Bending damping. Increase it to damp fold oscillations. + +Tuning Workflow +--------------- + +Use the following sequence when bringing up a new VBD task: + +1. Run one of the supported Franka deformable tasks and confirm your + installation, visualizer, and deformable rendering path work. +2. Add a task-specific VBD or coupled VBD preset copied from the closest + supported task. +3. Run a small visual smoke test with ``--num_envs 1`` before training. +4. Tune deformable material stiffness and damping until the object deforms in + the expected range without rigid contact. +5. Increase ``num_substeps`` or decrease ``dt`` if the object is unstable before + increasing stiffness further. +6. Increase :attr:`~isaaclab_contrib.deformable.VBDSolverCfg.iterations` when + contacts or stiff materials do not converge within a substep. +7. Tune :attr:`~isaaclab_contrib.deformable.NewtonModelCfg.soft_contact_ke` to + reduce rigid/deformable clipping, then tune + :attr:`~isaaclab_contrib.deformable.NewtonModelCfg.soft_contact_mu` for grip + and :attr:`~isaaclab_contrib.deformable.NewtonModelCfg.soft_contact_kd` for + chatter. +8. Enable self-contact only after body-particle contact is stable, then tune + ``particle_self_contact_radius`` for active self-contact thickness, + ``particle_self_contact_margin`` for missed contacts, and + ``particle_collision_detection_interval`` for detection frequency. +9. Increase ``num_envs`` and profile only after the single-environment scene is + stable. + + +Symptoms and First Parameters to Check +-------------------------------------- + +.. list-table:: + :header-rows: 1 + :widths: 35 65 + + * - Symptom + - First parameters to check + * - Rigid bodies visibly clip through the deformable. + - Increase ``soft_contact_ke``, VBD ``iterations``, ``num_substeps``, or the deformable material ``particle_radius``. + * - The robot cannot lift the deformable. + - Use ``coupling_mode="two_way"``, then increase ``soft_contact_mu`` and rigid-side ``shape_material_mu``. Also check gripper actuator stiffness and effort limits. + * - The deformable barely deforms. + - Reduce material stiffness, ``soft_contact_ke``, or shape contact stiffness. + * - Contact chatters or bounces. + - Increase ``soft_contact_kd`` or material damping, and consider using more substeps. + * - Cloth passes through itself. + - Enable ``particle_enable_self_contact``, increase ``particle_self_contact_radius`` if the active self-contact thickness is too small, increase ``particle_self_contact_margin`` if contacts are missed, and use a positive ``particle_collision_detection_interval``. + * - Self-contact is too expensive. + - Increase ``particle_collision_detection_interval``, reduce mesh resolution, or disable self-contact until the rest of the scene is tuned. + +For implementation details of the VBD and coupled solver managers, see +:doc:`newton-manager-abstraction`. diff --git a/docs/source/overview/core-concepts/physical-backends/newton/warp-environments.rst b/docs/source/overview/core-concepts/physical-backends/newton/warp-environments.rst index a5dcbd64872..3c0e1ba7d6b 100644 --- a/docs/source/overview/core-concepts/physical-backends/newton/warp-environments.rst +++ b/docs/source/overview/core-concepts/physical-backends/newton/warp-environments.rst @@ -321,11 +321,4 @@ Migrating Existing Environments For step-by-step instructions on porting an existing stable env (or writing a new warp env from scratch) — covering project layout, the kernel + launch pattern shared by observations / rewards / events / terminations / actions, capture-safety rules, and -parity testing — see :doc:`warp-env-migration` below. - - -.. toctree:: - :maxdepth: 2 - :hidden: - - warp-env-migration +parity testing — see :doc:`warp-env-migration`. diff --git a/docs/source/overview/core-concepts/renderers.rst b/docs/source/overview/core-concepts/renderers.rst index f04b476330b..b4e7c2a6454 100644 --- a/docs/source/overview/core-concepts/renderers.rst +++ b/docs/source/overview/core-concepts/renderers.rst @@ -32,6 +32,11 @@ Choosing a renderer backend .. note:: + Visualization markers are not yet supported by Newton-based renderer backends, + including the Newton Warp renderer. Use an RTX-based renderer, such as the + Isaac RTX renderer or OVRTX renderer, when marker visualization is needed. + +.. note:: **Temporal information for camera-based RL.** Unlike RTX modes with temporal anti-aliasing (DLSS, DLAA, TAA), the Newton Warp renderer does not inject prior-frame information into the current image. Camera-control tasks that depend diff --git a/docs/source/overview/core-concepts/scene_data_providers.rst b/docs/source/overview/core-concepts/scene_data_providers.rst index a279735efb3..9562610e8b0 100644 --- a/docs/source/overview/core-concepts/scene_data_providers.rst +++ b/docs/source/overview/core-concepts/scene_data_providers.rst @@ -1,7 +1,7 @@ Scene Data Provider =================== -The :class:`~isaaclab.scene.scene_data_provider.SceneDataProvider` bridges physics simulation +The :class:`~isaaclab.scene_data.SceneDataProvider` bridges physics simulation backends and the visualizers/renderers that consume scene data. It exposes a single Warp-native read path for body transforms regardless of which physics backend (PhysX or Newton) is active, so renderers and visualizers can stay backend-agnostic. @@ -12,7 +12,7 @@ Overview Isaac Lab supports multiple physics backends (PhysX and Newton) and multiple visualizers (Omniverse Kit, Newton, Rerun, Viser). Each combination needs scene data to flow from the physics engine into the renderer or visualizer. The :class:`SceneDataProvider` owns this flow: -the physics manager provides a :class:`~isaaclab.physics.SceneDataBackend` that wraps its +the physics manager provides a :class:`~isaaclab.scene_data.SceneDataBackend` that wraps its native tensor views, and the provider handles format conversion and re-mapping on top of it. .. code-block:: python @@ -28,9 +28,9 @@ Architecture The system has three layers: -1. :class:`~isaaclab.physics.SceneDataBackend` — small interface implemented by each physics +1. :class:`~isaaclab.scene_data.SceneDataBackend` — small interface implemented by each physics manager. It exposes the backend's transform array directly as one of the - :class:`~isaaclab.physics.SceneDataFormat` Warp structs, plus the per-transform prim paths + :class:`~isaaclab.scene_data.SceneDataFormat` Warp structs, plus the per-transform prim paths and total count. There is no per-frame "update" call — the property accessors return live views into the underlying tensor each time they're read. @@ -40,7 +40,7 @@ The system has three layers: - :attr:`SceneDataBackend.transform_count` — number of transforms. - :attr:`SceneDataBackend.transform_paths` — list of USD prim paths, one per transform. -2. :class:`~isaaclab.scene.scene_data_provider.SceneDataProvider` — wraps a backend and offers +2. :class:`~isaaclab.scene_data.SceneDataProvider` — wraps a backend and offers format conversion plus index re-mapping: - :meth:`SceneDataProvider.get_transforms` — write the backend's transforms into a diff --git a/docs/source/overview/environments.rst b/docs/source/overview/environments.rst index 61a13d058ff..384f2dbff51 100644 --- a/docs/source/overview/environments.rst +++ b/docs/source/overview/environments.rst @@ -151,6 +151,9 @@ for the lift-cube environment: +-------------------------+------------------------------+-----------------------------------------------------------------------------+------------------------------+ | |lift-cube| | |lift-cube-link| | Pick a cube and bring it to a sampled target position with the Franka robot | | +-------------------------+------------------------------+-----------------------------------------------------------------------------+------------------------------+ + | |lift-soft-franka| | |lift-soft-franka-link| | Pick a deformable soft body and bring it to a sampled target position with | ``newton_mjwarp_vdb``, | + | | | the Franka robot | ``physx`` | + +-------------------------+------------------------------+-----------------------------------------------------------------------------+------------------------------+ | |stack-cube| | |stack-cube-link| | Stack three cubes (bottom to top: blue, red, green) with the Franka robot. | | | | | Blueprint env used for the NVIDIA Isaac GR00T blueprint for synthetic | | | | |stack-cube-bp-link| | manipulation motion generation | | @@ -245,6 +248,7 @@ for the lift-cube environment: .. |reach-ur10| image:: ../_static/tasks/manipulation/ur10_reach.jpg .. |deploy-reach-ur10e| image:: ../_static/tasks/manipulation/ur10e_reach.jpg .. |lift-cube| image:: ../_static/tasks/manipulation/franka_lift.jpg +.. |lift-soft-franka| image:: ../_static/newton/franka-mjwarp-vbd-coupling.png .. |cabi-franka| image:: ../_static/tasks/manipulation/franka_open_drawer.jpg .. |cube-allegro| image:: ../_static/tasks/manipulation/allegro_cube.jpg .. |cube-shadow| image:: ../_static/tasks/manipulation/shadow_cube.jpg @@ -272,6 +276,7 @@ for the lift-cube environment: .. |lift-cube-link| replace:: `Isaac-Lift-Cube-Franka-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/lift/config/franka/joint_pos_env_cfg.py>`__ .. |lift-cube-ik-abs-link| replace:: `Isaac-Lift-Cube-Franka-IK-Abs-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/lift/config/franka/ik_abs_env_cfg.py>`__ .. |lift-cube-ik-rel-link| replace:: `Isaac-Lift-Cube-Franka-IK-Rel-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/lift/config/franka/ik_rel_env_cfg.py>`__ +.. |lift-soft-franka-link| replace:: `Isaac-Lift-Soft-Franka-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/lift_franka_soft/franka_soft_env_cfg.py>`__ .. |cabi-franka-link| replace:: `Isaac-Open-Drawer-Franka-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/cabinet/config/franka/joint_pos_env_cfg.py>`__ .. |franka-direct-link| replace:: `Isaac-Franka-Cabinet-Direct-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/direct/franka_cabinet/franka_cabinet_env.py>`__ .. |cube-allegro-link| replace:: `Isaac-Repose-Cube-Allegro-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/inhand/config/allegro_hand/allegro_env_cfg.py>`__ @@ -1047,6 +1052,16 @@ inferencing, including reading from an already trained checkpoint and disabling - Manager Based - **rsl_rl** (PPO), **skrl** (PPO), **rl_games** (PPO), **sb3** (PPO) - + * - Isaac-Lift-Soft-Franka-v0 + - + - Manager Based + - **rsl_rl** (PPO) + - ``newton_mjwarp_vdb``, ``physx`` + * - Isaac-Lift-Cloth-Franka-v0 + - + - Manager Based + - **rsl_rl** (PPO) + - ``newton_mjwarp_vdb`` * - Isaac-Lift-Teddy-Bear-Franka-IK-Abs-v0 - - Manager Based diff --git a/docs/source/overview/showroom.rst b/docs/source/overview/showroom.rst index bb224837574..358a310e9e1 100644 --- a/docs/source/overview/showroom.rst +++ b/docs/source/overview/showroom.rst @@ -57,7 +57,7 @@ A few quick showroom scripts to run and checkout: :alt: Biped robots in Isaac Lab -- Spawn different deformable (soft) bodies and let them fall from a height: +- Spawn different deformable objects and let them fall from a height: .. tab-set:: :sync-group: os @@ -76,6 +76,9 @@ A few quick showroom scripts to run and checkout: isaaclab.bat -p scripts\demos\deformables.py + Add ``--backend newton`` to run the same demo with the experimental Newton + deformable backend. + .. image:: ../_static/demos/deformables.jpg :width: 100% :alt: Deformable primitive-shaped objects in Isaac Lab diff --git a/docs/source/tutorials/00_sim/spawn_prims.rst b/docs/source/tutorials/00_sim/spawn_prims.rst index 1c4555687a2..86672091b2f 100644 --- a/docs/source/tutorials/00_sim/spawn_prims.rst +++ b/docs/source/tutorials/00_sim/spawn_prims.rst @@ -139,7 +139,8 @@ default to the default values set by USD Physics. Lastly, we spawn a cuboid ``CuboidDeformable`` which contains deformable body physics properties. Unlike the rigid body simulation, a deformable body can have relative motion between its vertices. This is useful for simulating soft bodies like cloth, rubber, or jello. It is important to note that deformable bodies are only supported in -GPU simulation and require a mesh object to be spawned with the deformable body physics properties. +GPU simulation and require a mesh object to be spawned with deformable body physics properties and a deformable +physics material. This example uses the PhysX-specific deformable property and material cfgs. .. literalinclude:: ../../../../scripts/tutorials/00_sim/spawn_prims.py :language: python diff --git a/docs/source/tutorials/01_assets/run_deformable_object.rst b/docs/source/tutorials/01_assets/run_deformable_object.rst index 61092353cec..bb2607a2094 100644 --- a/docs/source/tutorials/01_assets/run_deformable_object.rst +++ b/docs/source/tutorials/01_assets/run_deformable_object.rst @@ -7,14 +7,17 @@ Interacting with a deformable object .. currentmodule:: isaaclab While deformable objects sometimes refer to a broader class of objects, such as cloths, fluids and soft bodies, -in PhysX, deformable objects are represented as either surface or volume deformables. Unlike rigid objects, soft bodies can deform -under external forces and collisions. In this tutorial, we will focus on volume deformable bodies. For an example of surface -deformables (cloth), see the deformable demo at ``scripts/demos/deformables.py``. - -Soft bodies are simulated using Finite Element Method (FEM) in PhysX. The volume deformable comprises of two tetrahedral -meshes -- a simulation mesh and a collision mesh. The simulation mesh is used to simulate the deformations of -the soft body, while the collision mesh is used to detect collisions with other objects in the scene. -For more details, please check the `PhysX documentation`_. +Isaac Lab represents deformable objects as either surface or volume deformables. Unlike rigid objects, soft bodies can +deform under external forces and collisions. In this tutorial, we focus on volume deformable bodies. For an example of +surface deformables (cloth), see the deformable demo at ``scripts/demos/deformables.py``. + +The deformable object API and schema define/modify functions are shared across backends, while deformable +property and material configuration classes are backend-specific. PhysX simulates soft bodies using the Finite +Element Method (FEM); the Newton experimental backend uses VBD-based deformable support from +:mod:`isaaclab_contrib.deformable`. +The volume deformable comprises of two tetrahedral meshes -- a simulation mesh and a collision mesh. The simulation +mesh is used to simulate the deformations of the soft body, while the collision mesh is used to detect collisions +with other objects in the scene. For PhysX-specific details, please check the `PhysX documentation`_. This tutorial shows how to interact with a deformable object in the simulation. We will spawn a set of soft cubes and see how to set their nodal positions and velocities, along with apply kinematic @@ -31,7 +34,7 @@ The tutorial corresponds to the ``run_deformable_object.py`` script in the ``scr .. literalinclude:: ../../../../scripts/tutorials/01_assets/run_deformable_object.py :language: python - :emphasize-lines: 68-82, 110-126, 131-139, 141-149 + :emphasize-lines: 65-98, 119-124, 126-135, 140-148, 150-158 :linenos: @@ -54,8 +57,10 @@ the :class:`assets.DeformableObject` class, it spawns the object and initializes when the simulation is played. .. note:: - The deformable object is only supported in GPU simulation and requires a mesh object to be spawned with the - deformable body physics properties on it. + Deformable objects require a mesh object to be spawned with backend-specific deformable body physics + properties and a matching deformable physics material. + Use ``--backend physx`` for the PhysX implementation or ``--backend newton`` for the experimental Newton + implementation. As seen in the rigid body tutorial, we can spawn the deformable object into the scene in a similar fashion by creating @@ -63,7 +68,7 @@ an instance of the :class:`assets.DeformableObject` class by passing the configu .. literalinclude:: ../../../../scripts/tutorials/01_assets/run_deformable_object.py :language: python - :start-at: # Create separate groups called "Origin0", "Origin1", ... + :start-at: # Create separate groups called "env_0", "env_1", ... :end-at: cube_object = DeformableObject(cfg=cfg) Running the simulation loop @@ -112,8 +117,8 @@ Stepping the simulation """"""""""""""""""""""" Deformable bodies support user-driven kinematic control where a user can specify position targets for some of -the mesh nodes while the rest of the nodes are simulated using the FEM solver. This `partial kinematic`_ control -is useful for applications where the user wants to interact with the deformable object in a controlled manner. +the mesh nodes while the rest of the nodes are simulated by the active deformable solver. This `partial kinematic`_ +control is useful for applications where the user wants to interact with the deformable object in a controlled manner. In this tutorial, we apply kinematic commands to two out of the four cubes in the scene. We set the position targets for the node at index 0 (bottom-left corner) to move the cube along the z-axis. @@ -162,6 +167,12 @@ Now that we have gone through the code, let's run the script and see the result: ./isaaclab.sh -p scripts/tutorials/01_assets/run_deformable_object.py --visualizer kit +To run the same tutorial with the experimental Newton deformable backend: + +.. code-block:: bash + + ./isaaclab.sh -p scripts/tutorials/01_assets/run_deformable_object.py --backend newton --visualizer kit + This should open a stage with a ground plane, lights, and several cubes. Two of the four cubes must be dropping from a height and settling on to the ground. Meanwhile the other two cubes must be moving along the z-axis. You diff --git a/scripts/demos/deformables.py b/scripts/demos/deformables.py index 71f40b5f9c4..9da0ba19659 100644 --- a/scripts/demos/deformables.py +++ b/scripts/demos/deformables.py @@ -21,6 +21,7 @@ # create argparser parser = argparse.ArgumentParser(description="This script demonstrates how to spawn deformable prims into the scene.") +parser.add_argument("--backend", type=str, default="physx", choices=["physx", "newton"], help="Physics backend.") # append AppLauncher cli args AppLauncher.add_app_launcher_args(parser) # demos should open Kit visualizer by default @@ -39,13 +40,27 @@ import torch import tqdm -# deformables supported in PhysX -from isaaclab_physx.assets import DeformableObject, DeformableObjectCfg -from isaaclab_physx.sim import DeformableBodyMaterialCfg, DeformableBodyPropertiesCfg, SurfaceDeformableBodyMaterialCfg - import isaaclab.sim as sim_utils +from isaaclab.assets import DeformableObject, DeformableObjectCfg from isaaclab.utils.assets import ISAACLAB_NUCLEUS_DIR +if args_cli.backend == "newton": + from isaaclab_newton.sim.schemas import NewtonDeformableBodyPropertiesCfg as DeformableBodyPropertiesCfg + from isaaclab_newton.sim.spawners.materials import ( + NewtonDeformableBodyMaterialCfg as VolumeDeformableMaterialCfg, + ) + from isaaclab_newton.sim.spawners.materials import ( + NewtonSurfaceDeformableBodyMaterialCfg as SurfaceDeformableMaterialCfg, + ) +else: + from isaaclab_physx.sim.schemas import PhysxDeformableBodyPropertiesCfg as DeformableBodyPropertiesCfg + from isaaclab_physx.sim.spawners.materials import ( + PhysxDeformableBodyMaterialCfg as VolumeDeformableMaterialCfg, + ) + from isaaclab_physx.sim.spawners.materials import ( + PhysxSurfaceDeformableBodyMaterialCfg as SurfaceDeformableMaterialCfg, + ) + def define_origins(num_origins: int, radius: float = 2.0, center_height: float = 3.0) -> list[list[float]]: """Defines origins distributed on the surface of a sphere, sampled according to a Fibonacci lattice. @@ -84,47 +99,47 @@ def design_scene() -> tuple[dict, list[list[float]]]: radius=0.4, deformable_props=DeformableBodyPropertiesCfg(), visual_material=sim_utils.PreviewSurfaceCfg(), - physics_material=DeformableBodyMaterialCfg(), + physics_material=VolumeDeformableMaterialCfg(), ) cfg_cuboid = sim_utils.MeshCuboidCfg( size=(0.6, 0.6, 0.6), deformable_props=DeformableBodyPropertiesCfg(), visual_material=sim_utils.PreviewSurfaceCfg(), - physics_material=DeformableBodyMaterialCfg(), + physics_material=VolumeDeformableMaterialCfg(), ) cfg_cylinder = sim_utils.MeshCylinderCfg( radius=0.25, height=0.5, deformable_props=DeformableBodyPropertiesCfg(), visual_material=sim_utils.PreviewSurfaceCfg(), - physics_material=DeformableBodyMaterialCfg(), + physics_material=VolumeDeformableMaterialCfg(), ) cfg_capsule = sim_utils.MeshCapsuleCfg( radius=0.35, height=0.5, deformable_props=DeformableBodyPropertiesCfg(), visual_material=sim_utils.PreviewSurfaceCfg(), - physics_material=DeformableBodyMaterialCfg(), + physics_material=VolumeDeformableMaterialCfg(), ) cfg_cone = sim_utils.MeshConeCfg( radius=0.35, height=0.75, deformable_props=DeformableBodyPropertiesCfg(), visual_material=sim_utils.PreviewSurfaceCfg(), - physics_material=DeformableBodyMaterialCfg(), + physics_material=VolumeDeformableMaterialCfg(), ) - cfg_cloth = sim_utils.MeshSquareCfg( - size=1.5, + cfg_cloth = sim_utils.MeshRectangleCfg( + size=(1.5, 1.0), resolution=(21, 21), deformable_props=DeformableBodyPropertiesCfg(), visual_material=sim_utils.PreviewSurfaceCfg(), - physics_material=SurfaceDeformableBodyMaterialCfg(), + physics_material=SurfaceDeformableMaterialCfg(), ) cfg_usd = sim_utils.UsdFileCfg( usd_path=f"{ISAACLAB_NUCLEUS_DIR}/Objects/Teddy_Bear/teddy_bear.usd", deformable_props=DeformableBodyPropertiesCfg(), visual_material=sim_utils.PreviewSurfaceCfg(), - physics_material=DeformableBodyMaterialCfg(), + physics_material=VolumeDeformableMaterialCfg(), scale=[0.05, 0.05, 0.05], ) # create a dictionary of all the objects to be spawned @@ -141,50 +156,48 @@ def design_scene() -> tuple[dict, list[list[float]]]: # Create separate groups of deformable objects origins = define_origins(num_origins=12, radius=1.5, center_height=2.0) print("[INFO]: Spawning objects...") - num_volumes = 0 - num_surfaces = 0 - # Iterate over all the origins and randomly spawn objects + # Iterate over all the origins, spawn objects, and create a view for all the deformables + # note: since we manually spawned random deformable meshes above, we don't need to + # specify the spawn configuration for the deformable object + scene_entities = {} for idx, origin in tqdm.tqdm(enumerate(origins), total=len(origins)): # randomly select an object to spawn obj_name = random.choice(list(objects_cfg.keys())) obj_cfg = objects_cfg[obj_name] - # randomize the young modulus - obj_cfg.physics_material.youngs_modulus = random.uniform(5e5, 1e8) - # higher mesh resolution causes instability at low stiffness - if obj_name in ["sphere", "capsule", "cloth", "usd"]: - obj_cfg.physics_material.youngs_modulus = random.uniform(1e8, 5e9) - # randomize the poisson's ratio - obj_cfg.physics_material.poissons_ratio = random.uniform(0.25, 0.45) + # randomize the deformable material stiffness + if args_cli.backend == "newton" and obj_name == "cloth": + obj_cfg.physics_material.tri_ke = random.uniform(5e3, 5e4) + obj_cfg.physics_material.tri_ka = random.uniform(5e3, 5e4) + else: + youngs_modulus = random.uniform(5e5, 1e8) + poissons_ratio = random.uniform(0.25, 0.45) + if args_cli.backend == "newton": + obj_cfg.physics_material.k_mu = youngs_modulus / (2.0 * (1.0 + poissons_ratio)) + obj_cfg.physics_material.k_lambda = ( + youngs_modulus * poissons_ratio / ((1.0 + poissons_ratio) * (1.0 - 2.0 * poissons_ratio)) + ) + else: + obj_cfg.physics_material.youngs_modulus = youngs_modulus + obj_cfg.physics_material.poissons_ratio = poissons_ratio # randomize the color obj_cfg.visual_material.diffuse_color = (random.random(), random.random(), random.random()) # spawn the object, separate groups for surface and volume deformables if obj_name in ["cloth"]: - obj_cfg.func(f"/World/Origin/Surface{idx:02d}", obj_cfg, translation=origin) - num_surfaces += 1 + prim_path = f"/World/Origin/Surface{idx:02d}" + cfg = DeformableObjectCfg( + prim_path=prim_path, + spawn=obj_cfg, + init_state=DeformableObjectCfg.InitialStateCfg(pos=origin), + ) + scene_entities[f"Surface{idx:02d}"] = DeformableObject(cfg=cfg) else: - obj_cfg.func(f"/World/Origin/Volume{idx:02d}", obj_cfg, translation=origin) - num_volumes += 1 - - # create a view for all the deformables, separate views for volume and surface deformables - # note: since we manually spawned random deformable meshes above, we don't need to - # specify the spawn configuration for the deformable object - scene_entities = {} - if num_volumes > 0: - cfg = DeformableObjectCfg( - prim_path="/World/Origin/Volume.*", - spawn=None, - init_state=DeformableObjectCfg.InitialStateCfg(), - ) - volume_deformable_object = DeformableObject(cfg=cfg) - scene_entities["volume_deformable_object"] = volume_deformable_object - if num_surfaces > 0: - cfg = DeformableObjectCfg( - prim_path="/World/Origin/Surface.*", - spawn=None, - init_state=DeformableObjectCfg.InitialStateCfg(), - ) - surface_deformable_object = DeformableObject(cfg=cfg) - scene_entities["surface_deformable_object"] = surface_deformable_object + prim_path = f"/World/Origin/Volume{idx:02d}" + cfg = DeformableObjectCfg( + prim_path=prim_path, + spawn=obj_cfg, + init_state=DeformableObjectCfg.InitialStateCfg(pos=origin), + ) + scene_entities[f"Volume{idx:02d}"] = DeformableObject(cfg=cfg) # return the scene information return scene_entities, origins @@ -224,7 +237,17 @@ def run_simulator(sim: sim_utils.SimulationContext, entities: dict[str, Deformab def main(): """Main function.""" # Initialize the simulation context - sim_cfg = sim_utils.SimulationCfg(dt=0.01, device=args_cli.device) + if args_cli.backend == "newton": + from isaaclab_newton.physics import NewtonCfg + + from isaaclab_contrib.deformable.newton_manager_cfg import VBDSolverCfg + + physics_cfg = NewtonCfg(solver_cfg=VBDSolverCfg(iterations=10), num_substeps=4) + else: + from isaaclab_physx.physics import PhysxCfg + + physics_cfg = PhysxCfg() + sim_cfg = sim_utils.SimulationCfg(dt=0.01, device=args_cli.device, physics=physics_cfg) sim = sim_utils.SimulationContext(sim_cfg) # Set main camera sim.set_camera_view([4.0, 4.0, 3.0], [0.5, 0.5, 0.0]) diff --git a/scripts/environments/state_machine/lift_franka_soft.py b/scripts/environments/state_machine/lift_franka_soft.py new file mode 100644 index 00000000000..a9561718a59 --- /dev/null +++ b/scripts/environments/state_machine/lift_franka_soft.py @@ -0,0 +1,382 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +""" +Script to demonstrate lifting a deformable object with a robotic arm. + +The state machine is implemented in the kernel function `infer_state_machine`. +It uses the `warp` library to run the state machine in parallel on the GPU. + +.. code-block:: bash + + ./isaaclab.sh -p scripts/environments/state_machine/lift_franka_soft.py + +""" + +"""Launch Omniverse Toolkit first.""" + +import argparse + +from isaaclab.app import AppLauncher + +# add argparse arguments +parser = argparse.ArgumentParser(description="Pick and lift a deformable with a robotic arm.") +parser.add_argument("--num_envs", type=int, default=1, help="Number of environments to simulate.") +parser.add_argument("--task", type=str, default="Isaac-Lift-Soft-Franka-v0", help="The task to run.") +parser.add_argument("--video", action="store_true", default=False, help="Record a video of the rollout.") +parser.add_argument("--video_length", type=int, default=500, help="Length of the recorded video (in env steps).") +parser.add_argument( + "--video_folder", + type=str, + default="videos/lift_franka_soft", + help="Directory to write recorded videos into.", +) +# append AppLauncher cli args +AppLauncher.add_app_launcher_args(parser) +# parse the arguments +args_cli = parser.parse_args() + +# RecordVideo needs an rgb_array render mode, which in turn requires cameras to be enabled. +if args_cli.video: + args_cli.enable_cameras = True + +# launch omniverse app +app_launcher = AppLauncher(args_cli) +simulation_app = app_launcher.app + +# disable metrics assembler due to scene graph instancing +from isaacsim.core.experimental.utils.app import enable_extension + +enable_extension("omni.usd.metrics.assembler.ui", enabled=False) + +"""Rest everything else.""" + +import os +from collections.abc import Sequence + +import gymnasium as gym +import torch +import warp as wp + +from isaaclab.assets.deformable_object.deformable_object_data import DeformableObjectData + +import isaaclab_tasks # noqa: F401 +from isaaclab_tasks.manager_based.manipulation.lift_franka_soft.franka_cloth_env_cfg import FrankaClothEnvCfg +from isaaclab_tasks.manager_based.manipulation.lift_franka_soft.franka_soft_env_cfg import FrankaSoftEnvCfg +from isaaclab_tasks.utils.parse_cfg import parse_env_cfg + +# initialize warp +wp.init() + + +class GripperState: + """States for the gripper.""" + + OPEN = wp.constant(1.0) + CLOSE = wp.constant(-1.0) + + +class PickSmState: + """States for the pick state machine.""" + + REST = wp.constant(0) + APPROACH_ABOVE_OBJECT = wp.constant(1) + APPROACH_OBJECT = wp.constant(2) + GRASP_OBJECT = wp.constant(3) + LIFT_OBJECT = wp.constant(4) + OPEN_GRIPPER = wp.constant(5) + + +@wp.func +def distance_below_threshold(current_pos: wp.vec3, desired_pos: wp.vec3, threshold: float) -> bool: + return wp.length(current_pos - desired_pos) < threshold + + +@wp.kernel +def infer_state_machine( + dt: wp.array(dtype=float), + sm_state: wp.array(dtype=int), + sm_wait_time: wp.array(dtype=float), + ee_pose: wp.array(dtype=wp.transform), + object_pose: wp.array(dtype=wp.transform), + des_object_pose: wp.array(dtype=wp.transform), + des_ee_pose: wp.array(dtype=wp.transform), + gripper_state: wp.array(dtype=float), + offset: wp.array(dtype=wp.transform), + position_threshold: float, +): + # retrieve thread id + tid = wp.tid() + # retrieve state machine state + state = sm_state[tid] + # decide next state + if state == PickSmState.REST: + des_ee_pose[tid] = ee_pose[tid] + gripper_state[tid] = GripperState.OPEN + # wait for a while + if sm_wait_time[tid] >= PickSmWaitTime.REST: + # move to next state and reset wait time + sm_state[tid] = PickSmState.APPROACH_ABOVE_OBJECT + sm_wait_time[tid] = 0.0 + elif state == PickSmState.APPROACH_ABOVE_OBJECT: + des_ee_pose[tid] = wp.transform_multiply(offset[tid], object_pose[tid]) + gripper_state[tid] = GripperState.OPEN + if distance_below_threshold( + wp.transform_get_translation(ee_pose[tid]), + wp.transform_get_translation(des_ee_pose[tid]), + position_threshold, + ): + # wait for a while + if sm_wait_time[tid] >= PickSmWaitTime.APPROACH_OBJECT: + # move to next state and reset wait time + sm_state[tid] = PickSmState.APPROACH_OBJECT + sm_wait_time[tid] = 0.0 + elif state == PickSmState.APPROACH_OBJECT: + des_ee_pose[tid] = object_pose[tid] + gripper_state[tid] = GripperState.OPEN + if distance_below_threshold( + wp.transform_get_translation(ee_pose[tid]), + wp.transform_get_translation(des_ee_pose[tid]), + position_threshold, + ): + # wait for a while + if sm_wait_time[tid] >= PickSmWaitTime.APPROACH_OBJECT: + # move to next state and reset wait time + sm_state[tid] = PickSmState.GRASP_OBJECT + sm_wait_time[tid] = 0.0 + elif state == PickSmState.GRASP_OBJECT: + des_ee_pose[tid] = object_pose[tid] + gripper_state[tid] = GripperState.CLOSE + # wait for a while + if sm_wait_time[tid] >= PickSmWaitTime.GRASP_OBJECT: + # move to next state and reset wait time + sm_state[tid] = PickSmState.LIFT_OBJECT + sm_wait_time[tid] = 0.0 + elif state == PickSmState.LIFT_OBJECT: + des_ee_pose[tid] = des_object_pose[tid] + gripper_state[tid] = GripperState.CLOSE + if distance_below_threshold( + wp.transform_get_translation(ee_pose[tid]), + wp.transform_get_translation(des_ee_pose[tid]), + position_threshold, + ): + # wait for a while + if sm_wait_time[tid] >= PickSmWaitTime.LIFT_OBJECT: + # move to next state and reset wait time + sm_state[tid] = PickSmState.OPEN_GRIPPER + sm_wait_time[tid] = 0.0 + elif state == PickSmState.OPEN_GRIPPER: + # des_ee_pose[tid] = object_pose[tid] + gripper_state[tid] = GripperState.OPEN + # wait for a while + if sm_wait_time[tid] >= PickSmWaitTime.OPEN_GRIPPER: + # move to next state and reset wait time + sm_state[tid] = PickSmState.OPEN_GRIPPER + sm_wait_time[tid] = 0.0 + # increment wait time + sm_wait_time[tid] = sm_wait_time[tid] + dt[tid] + + +class PickSmWaitTime: + """Additional wait times (in s) for states for before switching. + + Wait times are generous because the low-PD Franka takes a while to settle on each IK target. + """ + + REST = wp.constant(0.2) + APPROACH_ABOVE_OBJECT = wp.constant(1.0) + APPROACH_OBJECT = wp.constant(1.0) + GRASP_OBJECT = wp.constant(1.0) + LIFT_OBJECT = wp.constant(1.5) + OPEN_GRIPPER = wp.constant(0.0) + + +class PickAndLiftSm: + """A simple state machine in a robot's task space to pick and lift an object. + + The state machine is implemented as a warp kernel. It takes in the current state of + the robot's end-effector and the object, and outputs the desired state of the robot's + end-effector and the gripper. The state machine is implemented as a finite state + machine with the following states: + + 1. REST: The robot is at rest. + 2. APPROACH_ABOVE_OBJECT: The robot moves above the object. + 3. APPROACH_OBJECT: The robot moves to the object. + 4. GRASP_OBJECT: The robot grasps the object. + 5. LIFT_OBJECT: The robot lifts the object to the desired pose. This is the final state. + """ + + def __init__(self, dt: float, num_envs: int, device: torch.device | str = "cpu", position_threshold=0.03): + """Initialize the state machine. + + Args: + dt: The environment time step. + num_envs: The number of environments to simulate. + device: The device to run the state machine on. + """ + # save parameters + self.dt = float(dt) + self.num_envs = num_envs + self.device = device + self.position_threshold = position_threshold + # initialize state machine + self.sm_dt = torch.full((self.num_envs,), self.dt, device=self.device) + self.sm_state = torch.full((self.num_envs,), 0, dtype=torch.int32, device=self.device) + self.sm_wait_time = torch.zeros((self.num_envs,), device=self.device) + + # desired state + self.des_ee_pose = torch.zeros((self.num_envs, 7), device=self.device) + self.des_gripper_state = torch.full((self.num_envs,), 0.0, device=self.device) + + # approach above object offset + self.offset = torch.zeros((self.num_envs, 7), device=self.device) + self.offset[:, 2] = 0.1 + self.offset[:, -1] = 1.0 # warp expects quaternion as (x, y, z, w) + + # convert to warp + self.sm_dt_wp = wp.from_torch(self.sm_dt, wp.float32) + self.sm_state_wp = wp.from_torch(self.sm_state, wp.int32) + self.sm_wait_time_wp = wp.from_torch(self.sm_wait_time, wp.float32) + self.des_ee_pose_wp = wp.from_torch(self.des_ee_pose, wp.transform) + self.des_gripper_state_wp = wp.from_torch(self.des_gripper_state, wp.float32) + self.offset_wp = wp.from_torch(self.offset, wp.transform) + + def reset_idx(self, env_ids: Sequence[int] = None): + """Reset the state machine.""" + if env_ids is None: + env_ids = slice(None) + self.sm_state[env_ids] = 0 + self.sm_wait_time[env_ids] = 0.0 + + def compute(self, ee_pose: torch.Tensor, object_pose: torch.Tensor, des_object_pose: torch.Tensor): + """Compute the desired state of the robot's end-effector and the gripper.""" + + # convert to warp + ee_pose_wp = wp.from_torch(ee_pose.contiguous(), wp.transform) + object_pose_wp = wp.from_torch(object_pose.contiguous(), wp.transform) + des_object_pose_wp = wp.from_torch(des_object_pose.contiguous(), wp.transform) + + # run state machine + wp.launch( + kernel=infer_state_machine, + dim=self.num_envs, + inputs=[ + self.sm_dt_wp, + self.sm_state_wp, + self.sm_wait_time_wp, + ee_pose_wp, + object_pose_wp, + des_object_pose_wp, + self.des_ee_pose_wp, + self.des_gripper_state_wp, + self.offset_wp, + self.position_threshold, + ], + device=self.device, + ) + + # convert to torch + return torch.cat([self.des_ee_pose, self.des_gripper_state.unsqueeze(-1)], dim=-1) + + +def main(): + # create environment + render_mode = "rgb_array" if args_cli.video else None + if args_cli.task == "Isaac-Lift-Soft-Franka-v0": + # parse configuration + env_cfg: FrankaSoftEnvCfg = parse_env_cfg( + "Isaac-Lift-Soft-Franka-v0", + device=args_cli.device, + num_envs=args_cli.num_envs, + ) + env_cfg.viewer.eye = (2.1, 1.0, 1.3) + env = gym.make("Isaac-Lift-Soft-Franka-v0", cfg=env_cfg, render_mode=render_mode) + elif args_cli.task == "Isaac-Lift-Soft-Franka-Cloth-v0": + # parse configuration + env_cfg: FrankaClothEnvCfg = parse_env_cfg( + "Isaac-Lift-Cloth-Franka-v0", + device=args_cli.device, + num_envs=args_cli.num_envs, + ) + env_cfg.viewer.eye = (2.1, 1.0, 1.3) + env = gym.make("Isaac-Lift-Cloth-Franka-v0", cfg=FrankaClothEnvCfg(), render_mode=render_mode) + else: + raise ValueError(f"Unknown task: {args_cli.task}") + + # wrap for video recording + if args_cli.video: + video_folder = os.path.abspath(args_cli.video_folder) + os.makedirs(video_folder, exist_ok=True) + env = gym.wrappers.RecordVideo( + env, + video_folder=video_folder, + step_trigger=lambda step: step == 0, + video_length=args_cli.video_length, + disable_logger=True, + ) + print(f"[INFO] Recording video to {video_folder} (length={args_cli.video_length} steps)") + + # reset environment at start + env.reset() + + # create action buffers (position + quaternion) + actions = torch.zeros(env.unwrapped.action_space.shape, device=env.unwrapped.device) + actions[:, 3] = 1.0 + # desired rotation after grasping + desired_orientation = torch.zeros((env.unwrapped.num_envs, 4), device=env.unwrapped.device) + desired_orientation[:, 0] = 1.0 + + # Top-down approach: identity quaternion (wxyz, w=1) aligns panda_hand with the Franka root, + # giving the canonical top-down grasp pose. The bar lies along world-X, so the gripper + # closes across its short side without any wrist twist. + object_grasp_orientation = torch.zeros((env.unwrapped.num_envs, 4), device=env.unwrapped.device) + object_grasp_orientation[:, 0] = 1.0 + # Grasp at the deformable's centre of mass. + object_local_grasp_position = torch.tensor([0.0, 0.0, 0.0], device=env.unwrapped.device) + + # create state machine + pick_sm = PickAndLiftSm(env_cfg.sim.dt * env_cfg.decimation, env.unwrapped.num_envs, env.unwrapped.device) + + while simulation_app.is_running(): + # run everything in inference mode + with torch.inference_mode(): + # step environment + dones = env.step(actions)[-2] + + # observations + # -- end-effector frame + ee_frame_sensor = env.unwrapped.scene["ee_frame"] + tcp_rest_position = ( + ee_frame_sensor.data.target_pos_w.torch[..., 0, :].clone() - env.unwrapped.scene.env_origins + ) + tcp_rest_orientation = ee_frame_sensor.data.target_quat_w.torch[..., 0, :].clone() + # -- object frame + object_data: DeformableObjectData = env.unwrapped.scene["deformable"].data + object_position = object_data.root_pos_w.torch - env.unwrapped.scene.env_origins + object_position += object_local_grasp_position + + # -- target object frame + desired_position = env.unwrapped.command_manager.get_command("deformable_pose")[..., :3] + + # advance state machine + actions = pick_sm.compute( + torch.cat([tcp_rest_position, tcp_rest_orientation], dim=-1), + torch.cat([object_position, object_grasp_orientation], dim=-1), + torch.cat([desired_position, desired_orientation], dim=-1), + ) + + # reset state machine + if dones.any(): + pick_sm.reset_idx(dones.nonzero(as_tuple=False).squeeze(-1)) + + # close the environment + env.close() + + +if __name__ == "__main__": + # run the main function + main() + # close sim app + simulation_app.close() diff --git a/scripts/tutorials/00_sim/spawn_prims.py b/scripts/tutorials/00_sim/spawn_prims.py index 7c120bd308d..8d366c650e6 100644 --- a/scripts/tutorials/00_sim/spawn_prims.py +++ b/scripts/tutorials/00_sim/spawn_prims.py @@ -31,6 +31,9 @@ """Rest everything follows.""" +from isaaclab_physx.sim.schemas import PhysxDeformableBodyPropertiesCfg +from isaaclab_physx.sim.spawners.materials import PhysxDeformableBodyMaterialCfg + import isaaclab.sim as sim_utils from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR @@ -75,9 +78,9 @@ def design_scene(): # spawn a blue cuboid with deformable body cfg_cuboid_deformable = sim_utils.MeshCuboidCfg( size=(0.2, 0.5, 0.2), - deformable_props=sim_utils.DeformableBodyPropertiesCfg(), + deformable_props=PhysxDeformableBodyPropertiesCfg(), visual_material=sim_utils.PreviewSurfaceCfg(diffuse_color=(0.0, 0.0, 1.0)), - physics_material=sim_utils.DeformableBodyMaterialCfg(), + physics_material=PhysxDeformableBodyMaterialCfg(), ) cfg_cuboid_deformable.func("/World/Objects/CuboidDeformable", cfg_cuboid_deformable, translation=(0.15, 0.0, 2.0)) diff --git a/scripts/tutorials/01_assets/run_deformable_object.py b/scripts/tutorials/01_assets/run_deformable_object.py index 368fe7c3358..9d42a5301ba 100644 --- a/scripts/tutorials/01_assets/run_deformable_object.py +++ b/scripts/tutorials/01_assets/run_deformable_object.py @@ -22,6 +22,7 @@ # add argparse arguments parser = argparse.ArgumentParser(description="Tutorial on interacting with a deformable object.") +parser.add_argument("--backend", type=str, default="physx", choices=["physx", "newton"], help="Physics backend.") # append AppLauncher cli args AppLauncher.add_app_launcher_args(parser) # demos should open Kit visualizer by default @@ -36,14 +37,10 @@ """Rest everything follows.""" import torch -from isaaclab_physx.assets import DeformableObject, DeformableObjectCfg - -# deformables supported in PhysX -from isaaclab_physx.sim import DeformableBodyMaterialCfg, DeformableBodyPropertiesCfg import isaaclab.sim as sim_utils import isaaclab.utils.math as math_utils -from isaaclab.sim import SimulationContext +from isaaclab.assets import DeformableObject, DeformableObjectCfg def design_scene(): @@ -58,20 +55,43 @@ def design_scene(): # Create a dictionary for the scene entities scene_entities = {} - # Create separate groups called "Origin0", "Origin1", ... - # Each group will have a robot in it + # Create separate groups called "env_0", "env_1", ... + # Newton's scene loader requires the "env_\d+" naming convention to + # detect per-environment Xforms and replicate them as separate worlds. origins = [[0.25, 0.25, 0.0], [-0.25, 0.25, 0.0], [0.25, -0.25, 0.0], [-0.25, -0.25, 0.0]] for i, origin in enumerate(origins): - sim_utils.create_prim(f"/World/Origin{i}", "Xform", translation=origin) + sim_utils.create_prim(f"/World/env_{i}", "Xform", translation=origin) + + youngs_modulus = 1e5 + poissons_ratio = 0.4 + density = 500.0 + if args_cli.backend == "newton": + from isaaclab_newton.sim.schemas import NewtonDeformableBodyPropertiesCfg + from isaaclab_newton.sim.spawners.materials import NewtonDeformableBodyMaterialCfg + + deformable_props = NewtonDeformableBodyPropertiesCfg() + physics_material = NewtonDeformableBodyMaterialCfg( + k_mu=youngs_modulus / (2.0 * (1.0 + poissons_ratio)), + k_lambda=youngs_modulus * poissons_ratio / ((1.0 + poissons_ratio) * (1.0 - 2.0 * poissons_ratio)), + density=density, + ) + else: + from isaaclab_physx.sim.schemas import PhysxDeformableBodyPropertiesCfg + from isaaclab_physx.sim.spawners.materials import PhysxDeformableBodyMaterialCfg + + deformable_props = PhysxDeformableBodyPropertiesCfg(rest_offset=0.0, contact_offset=0.001) + physics_material = PhysxDeformableBodyMaterialCfg( + poissons_ratio=poissons_ratio, youngs_modulus=youngs_modulus, density=density + ) # 3D Deformable Object cfg = DeformableObjectCfg( - prim_path="/World/Origin.*/Cube", + prim_path="/World/env_.*/Cube", spawn=sim_utils.MeshCuboidCfg( size=(0.2, 0.2, 0.2), - deformable_props=DeformableBodyPropertiesCfg(), + deformable_props=deformable_props, visual_material=sim_utils.PreviewSurfaceCfg(diffuse_color=(0.5, 0.1, 0.0)), - physics_material=DeformableBodyMaterialCfg(poissons_ratio=0.4, youngs_modulus=1e5), + physics_material=physics_material, ), init_state=DeformableObjectCfg.InitialStateCfg(pos=(0.0, 0.0, 1.0)), debug_vis=True, @@ -155,8 +175,18 @@ def run_simulator(sim: sim_utils.SimulationContext, entities: dict, origins: tor def main(): """Main function.""" # Load kit helper - sim_cfg = sim_utils.SimulationCfg(dt=0.01, device=args_cli.device) - sim = SimulationContext(sim_cfg) + if args_cli.backend == "newton": + from isaaclab_newton.physics import NewtonCfg + + from isaaclab_contrib.deformable.newton_manager_cfg import VBDSolverCfg + + physics_cfg = NewtonCfg(solver_cfg=VBDSolverCfg(iterations=10), num_substeps=4) + else: + from isaaclab_physx.physics import PhysxCfg + + physics_cfg = PhysxCfg() + sim_cfg = sim_utils.SimulationCfg(dt=0.01, device=args_cli.device, physics=physics_cfg) + sim = sim_utils.SimulationContext(sim_cfg) # Set main camera sim.set_camera_view(eye=[2.0, 2.0, 2.0], target=[0.0, 0.0, 0.75]) # Design scene diff --git a/source/isaaclab/changelog.d/daniela-move-scene-data.major.rst b/source/isaaclab/changelog.d/daniela-move-scene-data.major.rst new file mode 100644 index 00000000000..39c979e086c --- /dev/null +++ b/source/isaaclab/changelog.d/daniela-move-scene-data.major.rst @@ -0,0 +1,23 @@ +Added +^^^^^ + +* Added :mod:`isaaclab.scene_data` sub-package consolidating + :class:`~isaaclab.scene_data.SceneDataProvider`, + :class:`~isaaclab.scene_data.SceneDataBackend`, and + :class:`~isaaclab.scene_data.SceneDataFormat` in a single import location. + +Changed +^^^^^^^ + +* **Breaking:** Moved :class:`~isaaclab.scene_data.SceneDataProvider` from + :mod:`isaaclab.scene.scene_data_provider` and + :class:`~isaaclab.scene_data.SceneDataBackend` / + :class:`~isaaclab.scene_data.SceneDataFormat` from :mod:`isaaclab.physics` + to the new :mod:`isaaclab.scene_data` sub-package. Update imports:: + + # before + from isaaclab.scene.scene_data_provider import SceneDataProvider + from isaaclab.physics import SceneDataBackend, SceneDataFormat + + # after + from isaaclab.scene_data import SceneDataProvider, SceneDataBackend, SceneDataFormat diff --git a/source/isaaclab/changelog.d/mtrepte-tiled_viz.rst b/source/isaaclab/changelog.d/mtrepte-tiled_viz.rst new file mode 100644 index 00000000000..d7a7dfe0f04 --- /dev/null +++ b/source/isaaclab/changelog.d/mtrepte-tiled_viz.rst @@ -0,0 +1,9 @@ +Added +^^^^^ + +* Added shared visualizer defaults for CLI-created visualizer configs. + +Fixed +^^^^^ + +* Fixed visualizer tiled camera defaults and camera pose helpers so tiled camera views remain opt-in and respect selected environment ids. diff --git a/source/isaaclab/changelog.d/mtrepte-update-debug-viz.rst b/source/isaaclab/changelog.d/mtrepte-update-debug-viz.rst new file mode 100644 index 00000000000..b7aa84db0df --- /dev/null +++ b/source/isaaclab/changelog.d/mtrepte-update-debug-viz.rst @@ -0,0 +1,9 @@ +Fixed +^^^^^ + +* Fixed visualization marker backend initialization so USD markers remain available during rendering even when standalone visualizers are not launched. + +Changed +^^^^^^^ + +* Removed temporary startup and runtime debug breadcrumbs from core simulation and environment setup logs. diff --git a/source/isaaclab/changelog.d/mym-deformable-backend-split.major.rst b/source/isaaclab/changelog.d/mym-deformable-backend-split.major.rst new file mode 100644 index 00000000000..1fd4b8967be --- /dev/null +++ b/source/isaaclab/changelog.d/mym-deformable-backend-split.major.rst @@ -0,0 +1,19 @@ +Added +^^^^^ + +* Added common deformable property and material base cfgs in :mod:`isaaclab.sim`. + +Changed +^^^^^^^ + +* Changed deformable spawners to accept backend-specific deformable property and + material cfgs. Use PhysX cfgs from :mod:`isaaclab_physx.sim` or Newton cfgs + from :mod:`isaaclab_newton.sim`. + +Deprecated +^^^^^^^^^^ + +* Deprecated generic deformable property and material cfgs in favor of + ``PhysxDeformableBodyPropertiesCfg``, ``PhysxDeformableBodyMaterialCfg``, + ``PhysxSurfaceDeformableBodyMaterialCfg``, ``NewtonDeformableBodyPropertiesCfg``, + ``NewtonDeformableBodyMaterialCfg``, and ``NewtonSurfaceDeformableBodyMaterialCfg``. diff --git a/source/isaaclab/changelog.d/mym-deformable_experimental.minor.rst b/source/isaaclab/changelog.d/mym-deformable_experimental.minor.rst new file mode 100644 index 00000000000..059828bf03a --- /dev/null +++ b/source/isaaclab/changelog.d/mym-deformable_experimental.minor.rst @@ -0,0 +1,34 @@ +Added +^^^^^ + +* Added backend-neutral deformable asset APIs, including + :class:`~isaaclab.assets.DeformableObject`, + :class:`~isaaclab.assets.DeformableObjectCfg`, and shared base/data classes. +* Added deformable body spawner, schema, and material APIs under + :mod:`isaaclab.sim`, including + :class:`~isaaclab.sim.DeformableObjectSpawnerCfg`, + :class:`~isaaclab.sim.DeformableBodyPropertiesCfg`, + :func:`~isaaclab.sim.define_deformable_body_properties`, + :func:`~isaaclab.sim.modify_deformable_body_properties`, + :class:`~isaaclab.sim.DeformableBodyMaterialCfg`, + :class:`~isaaclab.sim.SurfaceDeformableBodyMaterialCfg`, and + :func:`~isaaclab.sim.spawn_deformable_body_material`. +* Added ``pytetwild`` as a package dependency for tetrahedral mesh generation. +* Added deformable API, migration, and tutorial documentation for + backend-neutral imports and Newton backend selection. + +Changed +^^^^^^^ + +* Changed deformable demos and tutorials to use the backend-neutral + :mod:`isaaclab.assets` and :mod:`isaaclab.sim` APIs with selectable PhysX or + Newton backends. +* Changed USD spawning to support deformable objects whose USD assets contain + embedded tetrahedral mesh data. +* **Breaking:** Renamed :class:`~isaaclab.sim.spawners.meshes.MeshSquareCfg` + to :class:`~isaaclab.sim.spawners.meshes.MeshRectangleCfg` and + :func:`~isaaclab.sim.spawners.meshes.spawn_mesh_square` to + :func:`~isaaclab.sim.spawners.meshes.spawn_mesh_rectangle`. The ``size`` + attribute is now a ``tuple[float, float]`` of X- and Y-axis lengths instead + of a single edge length. Migrate ``MeshSquareCfg(size=s)`` to + ``MeshRectangleCfg(size=(s, s))``. diff --git a/source/isaaclab/docs/CHANGELOG.rst b/source/isaaclab/docs/CHANGELOG.rst index 93d4c67b949..68028a87110 100644 --- a/source/isaaclab/docs/CHANGELOG.rst +++ b/source/isaaclab/docs/CHANGELOG.rst @@ -272,9 +272,9 @@ Added ``JointTypeFloatingBase``, OCS2's ``generalizedCoordinatesNum = 6 + actuatedJointsNum``, iDynTree's ``getFreeFloatingMassMatrix`` returning ``(6 + dofs, 6 + dofs)``). -* Added :attr:`~isaaclab.scene.scene_data_provider.SceneDataProvider.usd_stage`, - :attr:`~isaaclab.scene.scene_data_provider.SceneDataProvider.num_envs`, and - :meth:`~isaaclab.scene.scene_data_provider.SceneDataProvider.get_camera_transforms` +* Added :attr:`~isaaclab.scene_data.SceneDataProvider.usd_stage`, + :attr:`~isaaclab.scene_data.SceneDataProvider.num_envs`, and + :meth:`~isaaclab.scene_data.SceneDataProvider.get_camera_transforms` so visualizers and renderers can pull stage-derived data through the same Warp-native provider that already exposes transforms. @@ -309,12 +309,12 @@ Changed COM-referenced form can read :attr:`body_com_jacobian_w`. * **Breaking:** :class:`~isaaclab.visualizers.base_visualizer.BaseVisualizer` subclasses now receive a - :class:`~isaaclab.scene.scene_data_provider.SceneDataProvider` in + :class:`~isaaclab.scene_data.SceneDataProvider` in :meth:`~isaaclab.visualizers.base_visualizer.BaseVisualizer.initialize` instead of the removed ``BaseSceneDataProvider``. Read environment count - from :attr:`~isaaclab.scene.scene_data_provider.SceneDataProvider.num_envs` + from :attr:`~isaaclab.scene_data.SceneDataProvider.num_envs` and call - :meth:`~isaaclab.scene.scene_data_provider.SceneDataProvider.get_camera_transforms` + :meth:`~isaaclab.scene_data.SceneDataProvider.get_camera_transforms` on the new provider; both replace the previous ``get_metadata()`` / ``get_camera_transforms()`` calls on the legacy interface. diff --git a/source/isaaclab/isaaclab/assets/__init__.pyi b/source/isaaclab/isaaclab/assets/__init__.pyi index 727eae33002..55b42b8819a 100644 --- a/source/isaaclab/isaaclab/assets/__init__.pyi +++ b/source/isaaclab/isaaclab/assets/__init__.pyi @@ -21,6 +21,11 @@ __all__ = [ "RigidObjectCollection", "RigidObjectCollectionCfg", "RigidObjectCollectionData", + "BaseDeformableObject", + "BaseDeformableObjectData", + "DeformableObject", + "DeformableObjectCfg", + "DeformableObjectData", ] from .articulation import ( @@ -46,3 +51,10 @@ from .rigid_object_collection import ( RigidObjectCollectionCfg, RigidObjectCollectionData, ) +from .deformable_object import ( + BaseDeformableObject, + BaseDeformableObjectData, + DeformableObject, + DeformableObjectCfg, + DeformableObjectData, +) diff --git a/source/isaaclab/isaaclab/assets/deformable_object/__init__.py b/source/isaaclab/isaaclab/assets/deformable_object/__init__.py new file mode 100644 index 00000000000..525b6ae5fd9 --- /dev/null +++ b/source/isaaclab/isaaclab/assets/deformable_object/__init__.py @@ -0,0 +1,10 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Sub-module for deformable object assets.""" + +from isaaclab.utils.module import lazy_export + +lazy_export() diff --git a/source/isaaclab/isaaclab/assets/deformable_object/__init__.pyi b/source/isaaclab/isaaclab/assets/deformable_object/__init__.pyi new file mode 100644 index 00000000000..ebe78d5b6db --- /dev/null +++ b/source/isaaclab/isaaclab/assets/deformable_object/__init__.pyi @@ -0,0 +1,18 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +from .base_deformable_object import BaseDeformableObject +from .base_deformable_object_data import BaseDeformableObjectData +from .deformable_object import DeformableObject +from .deformable_object_cfg import DeformableObjectCfg +from .deformable_object_data import DeformableObjectData + +__all__ = [ + "BaseDeformableObject", + "BaseDeformableObjectData", + "DeformableObject", + "DeformableObjectCfg", + "DeformableObjectData", +] diff --git a/source/isaaclab/isaaclab/assets/deformable_object/base_deformable_object.py b/source/isaaclab/isaaclab/assets/deformable_object/base_deformable_object.py new file mode 100644 index 00000000000..e21895a10bf --- /dev/null +++ b/source/isaaclab/isaaclab/assets/deformable_object/base_deformable_object.py @@ -0,0 +1,371 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +from __future__ import annotations + +import warnings +from abc import abstractmethod +from collections.abc import Sequence +from typing import TYPE_CHECKING + +import torch +import warp as wp + +import isaaclab.utils.math as math_utils +from isaaclab.assets.asset_base import AssetBase +from isaaclab.utils.warp import ProxyArray + +if TYPE_CHECKING: + from .base_deformable_object_data import BaseDeformableObjectData + from .deformable_object_cfg import DeformableObjectCfg + + +class BaseDeformableObject(AssetBase): + """Abstract base class for deformable object assets. + + Deformable objects are assets that can be deformed in the simulation. They are typically used for + soft bodies, such as stuffed animals, food items, and cloth. + + Unlike rigid object assets, deformable objects have a more complex structure and require additional + handling for simulation. The state of a deformable object comprises of its nodal positions and + velocities, and not the object's root position and orientation. The nodal positions and velocities + are in the simulation frame. + + Soft bodies can be `partially kinematic`_, where some nodes are driven by kinematic targets, and + the rest are simulated. The kinematic targets are the desired positions of the nodes, and the + simulation drives the nodes towards these targets. + + .. _partially kinematic: https://nvidia-omniverse.github.io/PhysX/physx/5.4.1/docs/SoftBodies.html#kinematic-soft-bodies + """ + + cfg: DeformableObjectCfg + """Configuration instance for the deformable object.""" + + __backend_name__: str = "base" + """The name of the backend for the deformable object.""" + + def __init__(self, cfg: DeformableObjectCfg): + """Initialize the deformable object. + + Args: + cfg: A configuration instance. + """ + super().__init__(cfg) + + """ + Properties + """ + + @property + @abstractmethod + def data(self) -> BaseDeformableObjectData: + """Data container for the deformable object.""" + raise NotImplementedError() + + @property + @abstractmethod + def num_instances(self) -> int: + """Number of instances of the asset.""" + raise NotImplementedError() + + @property + @abstractmethod + def num_bodies(self) -> int: + """Number of bodies in the asset. + + This is always 1 since each object is a single deformable body. + """ + raise NotImplementedError() + + @property + @abstractmethod + def max_sim_vertices_per_body(self) -> int: + """The maximum number of simulation mesh vertices per deformable body.""" + raise NotImplementedError() + + """ + Operations. + """ + + @abstractmethod + def reset(self, env_ids: Sequence[int] | None = None, env_mask: wp.array | None = None) -> None: + """Reset the deformable object. + + Args: + env_ids: Environment indices. If None, then all indices are used. + env_mask: Environment mask. If None, then all the instances are updated. + Shape is (num_instances,). + """ + raise NotImplementedError() + + @abstractmethod + def write_data_to_sim(self): + """Write data to the simulator.""" + raise NotImplementedError() + + @abstractmethod + def update(self, dt: float): + """Update the internal buffers. + + Args: + dt: The amount of time passed from last :meth:`update` call [s]. + """ + raise NotImplementedError() + + """ + Operations - Write to simulation (index variants). + """ + + def write_nodal_state_to_sim_index( + self, + nodal_state: torch.Tensor | wp.array | ProxyArray, + env_ids: Sequence[int] | torch.Tensor | wp.array | None = None, + full_data: bool = False, + ) -> None: + """Set the nodal state over selected environment indices into the simulation. + + The nodal state comprises of the nodal positions and velocities. Since these are nodes, + the velocity only has a translational component. All the quantities are in the simulation frame. + + Args: + nodal_state: Nodal state in simulation frame [m, m/s]. + Shape is (len(env_ids), max_sim_vertices_per_body, 6) + or (num_instances, max_sim_vertices_per_body, 6). + env_ids: Environment indices. If None, then all indices are used. + full_data: Whether to expect full data. Defaults to False. + """ + # Convert array wrappers to torch for slicing into position and velocity views. + if isinstance(nodal_state, ProxyArray): + nodal_state = nodal_state.torch + elif isinstance(nodal_state, wp.array): + nodal_state = wp.to_torch(nodal_state) + # set into simulation + self.write_nodal_pos_to_sim_index(nodal_state[..., :3], env_ids=env_ids, full_data=full_data) + self.write_nodal_velocity_to_sim_index(nodal_state[..., 3:], env_ids=env_ids, full_data=full_data) + + @abstractmethod + def write_nodal_pos_to_sim_index( + self, + nodal_pos: torch.Tensor | wp.array | ProxyArray, + env_ids: Sequence[int] | torch.Tensor | wp.array | None = None, + full_data: bool = False, + ) -> None: + """Set the nodal positions over selected environment indices into the simulation. + + Args: + nodal_pos: Nodal positions in simulation frame [m]. + Shape is (len(env_ids), max_sim_vertices_per_body, 3) + or (num_instances, max_sim_vertices_per_body, 3). + env_ids: Environment indices. If None, then all indices are used. + full_data: Whether to expect full data. Defaults to False. + """ + raise NotImplementedError() + + @abstractmethod + def write_nodal_velocity_to_sim_index( + self, + nodal_vel: torch.Tensor | wp.array | ProxyArray, + env_ids: Sequence[int] | torch.Tensor | wp.array | None = None, + full_data: bool = False, + ) -> None: + """Set the nodal velocity over selected environment indices into the simulation. + + Args: + nodal_vel: Nodal velocities in simulation frame [m/s]. + Shape is (len(env_ids), max_sim_vertices_per_body, 3) + or (num_instances, max_sim_vertices_per_body, 3). + env_ids: Environment indices. If None, then all indices are used. + full_data: Whether to expect full data. Defaults to False. + """ + raise NotImplementedError() + + @abstractmethod + def write_nodal_kinematic_target_to_sim_index( + self, + targets: torch.Tensor | wp.array | ProxyArray, + env_ids: Sequence[int] | torch.Tensor | wp.array | None = None, + full_data: bool = False, + ) -> None: + """Set the kinematic targets of the simulation mesh for the deformable bodies using indices. + + The kinematic targets comprise of individual nodal positions of the simulation mesh + for the deformable body and a flag indicating whether the node is kinematically driven or not. + The positions are in the simulation frame. + + .. note:: + The flag is set to 0.0 for kinematically driven nodes and 1.0 for free nodes. + + Args: + targets: The kinematic targets comprising of nodal positions and flags [m]. + Shape is (len(env_ids), max_sim_vertices_per_body, 4) + or (num_instances, max_sim_vertices_per_body, 4). + env_ids: Environment indices. If None, then all indices are used. + full_data: Whether to expect full data. Defaults to False. + """ + raise NotImplementedError() + + """ + Operations - Write to simulation (mask variants). + """ + + def write_nodal_state_to_sim_mask( + self, + nodal_state: torch.Tensor | wp.array | ProxyArray, + env_mask: wp.array | None = None, + ) -> None: + """Set the nodal state over selected environment mask into the simulation. + + Args: + nodal_state: Nodal state in simulation frame [m, m/s]. + Shape is (num_instances, max_sim_vertices_per_body, 6). + env_mask: Environment mask. If None, then all indices are used. + """ + if env_mask is not None: + env_ids = wp.nonzero(env_mask) + else: + env_ids = self._ALL_INDICES + self.write_nodal_state_to_sim_index(nodal_state, env_ids=env_ids, full_data=True) + + def write_nodal_pos_to_sim_mask( + self, + nodal_pos: torch.Tensor | wp.array | ProxyArray, + env_mask: wp.array | None = None, + ) -> None: + """Set the nodal positions over selected environment mask into the simulation. + + Args: + nodal_pos: Nodal positions in simulation frame [m]. + Shape is (num_instances, max_sim_vertices_per_body, 3). + env_mask: Environment mask. If None, then all indices are used. + """ + if env_mask is not None: + env_ids = wp.nonzero(env_mask) + else: + env_ids = self._ALL_INDICES + self.write_nodal_pos_to_sim_index(nodal_pos, env_ids=env_ids, full_data=True) + + def write_nodal_velocity_to_sim_mask( + self, + nodal_vel: torch.Tensor | wp.array | ProxyArray, + env_mask: wp.array | None = None, + ) -> None: + """Set the nodal velocity over selected environment mask into the simulation. + + Args: + nodal_vel: Nodal velocities in simulation frame [m/s]. + Shape is (num_instances, max_sim_vertices_per_body, 3). + env_mask: Environment mask. If None, then all indices are used. + """ + if env_mask is not None: + env_ids = wp.nonzero(env_mask) + else: + env_ids = self._ALL_INDICES + self.write_nodal_velocity_to_sim_index(nodal_vel, env_ids=env_ids, full_data=True) + + def write_nodal_kinematic_target_to_sim_mask( + self, + targets: torch.Tensor | wp.array | ProxyArray, + env_mask: wp.array | None = None, + ) -> None: + """Set the kinematic targets of the simulation mesh for the deformable bodies using mask. + + Args: + targets: The kinematic targets comprising of nodal positions and flags [m]. + Shape is (num_instances, max_sim_vertices_per_body, 4). + env_mask: Environment mask. If None, then all indices are used. + """ + if env_mask is not None: + env_ids = wp.nonzero(env_mask) + else: + env_ids = self._ALL_INDICES + self.write_nodal_kinematic_target_to_sim_index(targets, env_ids=env_ids, full_data=True) + + """ + Operations - Deprecated wrappers. + """ + + def write_nodal_state_to_sim( + self, + nodal_state: torch.Tensor | wp.array | ProxyArray, + env_ids: Sequence[int] | torch.Tensor | wp.array | None = None, + ) -> None: + """Deprecated. Please use :meth:`write_nodal_state_to_sim_index` instead.""" + warnings.warn( + "The method 'write_nodal_state_to_sim' is deprecated. Please use 'write_nodal_state_to_sim_index' instead.", + DeprecationWarning, + stacklevel=2, + ) + self.write_nodal_state_to_sim_index(nodal_state, env_ids=env_ids) + + def write_nodal_kinematic_target_to_sim( + self, + targets: torch.Tensor | wp.array | ProxyArray, + env_ids: Sequence[int] | torch.Tensor | wp.array | None = None, + ) -> None: + """Deprecated. Please use :meth:`write_nodal_kinematic_target_to_sim_index` instead.""" + warnings.warn( + "The method 'write_nodal_kinematic_target_to_sim' is deprecated." + " Please use 'write_nodal_kinematic_target_to_sim_index' instead.", + DeprecationWarning, + stacklevel=2, + ) + self.write_nodal_kinematic_target_to_sim_index(targets, env_ids=env_ids) + + def write_nodal_pos_to_sim( + self, + nodal_pos: torch.Tensor | wp.array | ProxyArray, + env_ids: Sequence[int] | torch.Tensor | wp.array | None = None, + ) -> None: + """Deprecated. Please use :meth:`write_nodal_pos_to_sim_index` instead.""" + warnings.warn( + "The method 'write_nodal_pos_to_sim' is deprecated. Please use 'write_nodal_pos_to_sim_index' instead.", + DeprecationWarning, + stacklevel=2, + ) + self.write_nodal_pos_to_sim_index(nodal_pos, env_ids=env_ids) + + def write_nodal_velocity_to_sim( + self, + nodal_vel: torch.Tensor | wp.array | ProxyArray, + env_ids: Sequence[int] | torch.Tensor | wp.array | None = None, + ) -> None: + """Deprecated. Please use :meth:`write_nodal_velocity_to_sim_index` instead.""" + warnings.warn( + "The method 'write_nodal_velocity_to_sim' is deprecated." + " Please use 'write_nodal_velocity_to_sim_index' instead.", + DeprecationWarning, + stacklevel=2, + ) + self.write_nodal_velocity_to_sim_index(nodal_vel, env_ids=env_ids) + + """ + Operations - Helper. + """ + + def transform_nodal_pos( + self, nodal_pos: torch.Tensor, pos: torch.Tensor | None = None, quat: torch.Tensor | None = None + ) -> torch.Tensor: + """Transform the nodal positions based on the pose transformation. + + This function computes the transformation of the nodal positions based on the pose transformation. + It multiplies the nodal positions with the rotation matrix of the pose and adds the translation. + Internally, it calls the :meth:`isaaclab.utils.math.transform_points` function. + + Args: + nodal_pos: The nodal positions in the simulation frame [m]. + Shape is (N, max_sim_vertices_per_body, 3). + pos: The position transformation [m]. Shape is (N, 3). + Defaults to None, in which case the position is assumed to be zero. + quat: The orientation transformation as quaternion (x, y, z, w). Shape is (N, 4). + Defaults to None, in which case the orientation is assumed to be identity. + + Returns: + The transformed nodal positions [m]. Shape is (N, max_sim_vertices_per_body, 3). + """ + # offset the nodal positions to center them around the origin + mean_nodal_pos = nodal_pos.mean(dim=1, keepdim=True) + nodal_pos = nodal_pos - mean_nodal_pos + # transform the nodal positions based on the pose around the origin + return math_utils.transform_points(nodal_pos, pos, quat) + mean_nodal_pos diff --git a/source/isaaclab/isaaclab/assets/deformable_object/base_deformable_object_data.py b/source/isaaclab/isaaclab/assets/deformable_object/base_deformable_object_data.py new file mode 100644 index 00000000000..e0b1ae49751 --- /dev/null +++ b/source/isaaclab/isaaclab/assets/deformable_object/base_deformable_object_data.py @@ -0,0 +1,136 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +from __future__ import annotations + +from abc import ABC, abstractmethod + +from isaaclab.utils.warp import ProxyArray + + +class BaseDeformableObjectData(ABC): + """Abstract data container for a deformable object. + + This class defines the interface for deformable object data in the simulation. + The data includes the nodal states of the root deformable body in the object. + The data is stored in the simulation world frame unless otherwise specified. + + The data is lazily updated, meaning that the data is only updated when it is accessed. + This is useful when the data is expensive to compute or retrieve. The data is updated + when the timestamp of the buffer is older than the current simulation timestamp. + """ + + def __init__(self, device: str): + """Initialize the deformable object data. + + Args: + device: The device used for processing. + """ + self.device = device + # Set initial time stamp + self._sim_timestamp = 0.0 + + def update(self, dt: float): + """Update the data for the deformable object. + + Args: + dt: The time step for the update [s]. This must be a positive value. + """ + # update the simulation timestamp + self._sim_timestamp += dt + + ## + # Defaults. + ## + + default_nodal_state_w: ProxyArray | None = None + """Default nodal state ``[nodal_pos, nodal_vel]`` in simulation world frame. + + Shape is (num_instances, max_sim_vertices_per_body), dtype ``vec6f``. + Use :attr:`ProxyArray.warp` for the underlying :class:`warp.array` or + :attr:`ProxyArray.torch` for a cached zero-copy :class:`torch.Tensor` view. + """ + + ## + # Kinematic commands. + ## + + nodal_kinematic_target: ProxyArray | None = None + """Simulation mesh kinematic targets for the deformable bodies. + + Shape is (num_instances, max_sim_vertices_per_body), dtype ``wp.vec4f``. + Use :attr:`ProxyArray.warp` for the underlying :class:`warp.array` or + :attr:`ProxyArray.torch` for a cached zero-copy :class:`torch.Tensor` view. + + The kinematic targets are used to drive the simulation mesh vertices to the target positions. + The targets are stored as (x, y, z, is_not_kinematic) where "is_not_kinematic" is a binary + flag indicating whether the vertex is kinematic or not. The flag is set to 0 for kinematic vertices + and 1 for non-kinematic vertices. + """ + + ## + # Properties. + ## + + @property + @abstractmethod + def nodal_pos_w(self) -> ProxyArray: + """Nodal positions in simulation world frame [m]. + + Shape is (num_instances, max_sim_vertices_per_body), dtype ``wp.vec3f``. + Use :attr:`ProxyArray.warp` for the underlying :class:`warp.array` or + :attr:`ProxyArray.torch` for a cached zero-copy :class:`torch.Tensor` view. + """ + raise NotImplementedError() + + @property + @abstractmethod + def nodal_vel_w(self) -> ProxyArray: + """Nodal velocities in simulation world frame [m/s]. + + Shape is (num_instances, max_sim_vertices_per_body), dtype ``wp.vec3f``. + Use :attr:`ProxyArray.warp` for the underlying :class:`warp.array` or + :attr:`ProxyArray.torch` for a cached zero-copy :class:`torch.Tensor` view. + """ + raise NotImplementedError() + + @property + @abstractmethod + def nodal_state_w(self) -> ProxyArray: + """Nodal state ``[nodal_pos, nodal_vel]`` in simulation world frame [m, m/s]. + + Shape is (num_instances, max_sim_vertices_per_body), dtype ``vec6f``. + Use :attr:`ProxyArray.warp` for the underlying :class:`warp.array` or + :attr:`ProxyArray.torch` for a cached zero-copy :class:`torch.Tensor` view. + """ + raise NotImplementedError() + + ## + # Derived properties. + ## + + @property + @abstractmethod + def root_pos_w(self) -> ProxyArray: + """Root position from nodal positions of the simulation mesh for the deformable bodies + in simulation world frame [m]. Shape is (num_instances,) vec3f. + + This quantity is computed as the mean of the nodal positions. + Use :attr:`ProxyArray.warp` for the underlying :class:`warp.array` or + :attr:`ProxyArray.torch` for a cached zero-copy :class:`torch.Tensor` view. + """ + raise NotImplementedError() + + @property + @abstractmethod + def root_vel_w(self) -> ProxyArray: + """Root velocity from vertex velocities for the deformable bodies in simulation + world frame [m/s]. Shape is (num_instances,) vec3f. + + This quantity is computed as the mean of the nodal velocities. + Use :attr:`ProxyArray.warp` for the underlying :class:`warp.array` or + :attr:`ProxyArray.torch` for a cached zero-copy :class:`torch.Tensor` view. + """ + raise NotImplementedError() diff --git a/source/isaaclab/isaaclab/assets/deformable_object/deformable_object.py b/source/isaaclab/isaaclab/assets/deformable_object/deformable_object.py new file mode 100644 index 00000000000..3bd41c8728b --- /dev/null +++ b/source/isaaclab/isaaclab/assets/deformable_object/deformable_object.py @@ -0,0 +1,27 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from isaaclab.utils.backend_utils import FactoryBase + +from .base_deformable_object import BaseDeformableObject +from .base_deformable_object_data import BaseDeformableObjectData + +if TYPE_CHECKING: + from isaaclab_physx.assets.deformable_object import DeformableObject as PhysXDeformableObject + from isaaclab_physx.assets.deformable_object import DeformableObjectData as PhysXDeformableObjectData + + +class DeformableObject(FactoryBase, BaseDeformableObject): + """Factory for creating deformable object instances.""" + + data: BaseDeformableObjectData | PhysXDeformableObjectData + + def __new__(cls, *args, **kwargs) -> BaseDeformableObject | PhysXDeformableObject: + """Create a new instance of a deformable object based on the backend.""" + return super().__new__(cls, *args, **kwargs) diff --git a/source/isaaclab_physx/isaaclab_physx/assets/deformable_object/deformable_object_cfg.py b/source/isaaclab/isaaclab/assets/deformable_object/deformable_object_cfg.py similarity index 100% rename from source/isaaclab_physx/isaaclab_physx/assets/deformable_object/deformable_object_cfg.py rename to source/isaaclab/isaaclab/assets/deformable_object/deformable_object_cfg.py diff --git a/source/isaaclab/isaaclab/assets/deformable_object/deformable_object_data.py b/source/isaaclab/isaaclab/assets/deformable_object/deformable_object_data.py new file mode 100644 index 00000000000..35567aceb84 --- /dev/null +++ b/source/isaaclab/isaaclab/assets/deformable_object/deformable_object_data.py @@ -0,0 +1,25 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from isaaclab.utils.backend_utils import FactoryBase + +from .base_deformable_object_data import BaseDeformableObjectData + +if TYPE_CHECKING: + from isaaclab_physx.assets.deformable_object.deformable_object_data import ( + DeformableObjectData as PhysXDeformableObjectData, + ) + + +class DeformableObjectData(FactoryBase): + """Factory for creating deformable object data instances.""" + + def __new__(cls, *args, **kwargs) -> BaseDeformableObjectData | PhysXDeformableObjectData: + """Create a new instance of a deformable object data based on the backend.""" + return super().__new__(cls, *args, **kwargs) diff --git a/source/isaaclab/isaaclab/cli/commands/install.py b/source/isaaclab/isaaclab/cli/commands/install.py index 90fa929b69e..2953919c442 100644 --- a/source/isaaclab/isaaclab/cli/commands/install.py +++ b/source/isaaclab/isaaclab/cli/commands/install.py @@ -649,6 +649,8 @@ def _install_extra_feature(feature_name: str, selector: str = "") -> None: "websockets", "viser", "imgui_bundle", + "attr", + "attrs", ] """Package directory names in Isaac Sim prebundle directories to repoint. diff --git a/source/isaaclab/isaaclab/envs/direct_marl_env.py b/source/isaaclab/isaaclab/envs/direct_marl_env.py index 824dda6d015..2119fbe4d63 100644 --- a/source/isaaclab/isaaclab/envs/direct_marl_env.py +++ b/source/isaaclab/isaaclab/envs/direct_marl_env.py @@ -147,6 +147,7 @@ def _init_sim(self, render_mode: str | None = None, **kwargs): self.scene = InteractiveScene(self.cfg.scene) self._setup_scene() self.scene.initialize_renderers() + self.sim.register_interactive_scene(self.scene) print("[INFO]: Scene manager: ", self.scene) # set up camera viewport controller diff --git a/source/isaaclab/isaaclab/envs/direct_rl_env.py b/source/isaaclab/isaaclab/envs/direct_rl_env.py index 650568bbb1e..7e931c8eabf 100644 --- a/source/isaaclab/isaaclab/envs/direct_rl_env.py +++ b/source/isaaclab/isaaclab/envs/direct_rl_env.py @@ -152,6 +152,7 @@ def _init_sim(self, render_mode: str | None = None, **kwargs): self.scene = InteractiveScene(self.cfg.scene) self._setup_scene() self.scene.initialize_renderers() + self.sim.register_interactive_scene(self.scene) print("[INFO]: Scene manager: ", self.scene) # set up camera viewport controller diff --git a/source/isaaclab/isaaclab/envs/manager_based_env.py b/source/isaaclab/isaaclab/envs/manager_based_env.py index f89101dae07..95011738aed 100644 --- a/source/isaaclab/isaaclab/envs/manager_based_env.py +++ b/source/isaaclab/isaaclab/envs/manager_based_env.py @@ -171,6 +171,7 @@ def _init_sim(self): with use_stage(self.sim.stage): self.scene = InteractiveScene(self.cfg.scene) self.scene.initialize_renderers() + self.sim.register_interactive_scene(self.scene) print("[INFO]: Scene manager: ", self.scene) # set up camera viewport controller diff --git a/source/isaaclab/isaaclab/envs/utils/camera_view.py b/source/isaaclab/isaaclab/envs/utils/camera_view.py new file mode 100644 index 00000000000..7b7c093c3c8 --- /dev/null +++ b/source/isaaclab/isaaclab/envs/utils/camera_view.py @@ -0,0 +1,278 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Helpers for visualizer and recorder camera image views.""" + +from __future__ import annotations + +import math +import random +from typing import Any + +import torch +import warp as wp + +import isaaclab.sim as sim_utils +from isaaclab.sensors.camera import Camera, CameraCfg + +_GENERATED_CAMERA_NAME = "VisualizerCamera" +VISUALIZER_TILED_CAMERA_MAX_TILES = 100 + + +def resolve_tiled_env_indices( + num_envs: int, + tiled_cam_num: int, + env_indices: list[int] | None, + max_tiles: int | None = None, + sample_from: list[int] | None = None, +) -> list[int]: + """Resolve env ids for tiled camera view once at visualizer initialization.""" + if num_envs <= 0: + return [] + if env_indices is not None: + max_count = min(max(1, int(tiled_cam_num)), num_envs) + if max_tiles is not None: + max_count = min(max_count, max(1, int(max_tiles))) + return [idx for idx in env_indices if 0 <= int(idx) < num_envs][:max_count] + candidates = [ + idx for idx in (sample_from if sample_from is not None else range(num_envs)) if 0 <= int(idx) < num_envs + ] + if not candidates: + return [] + max_count = min(max(1, int(tiled_cam_num)), len(candidates)) + if max_tiles is not None: + max_count = min(max_count, max(1, int(max_tiles))) + return sorted(random.sample(candidates, max_count)) + + +def resolve_mono_env_index(num_envs: int) -> list[int]: + """Return env_0 for mono sensor camera views.""" + return [0] if num_envs > 0 else [] + + +def env_path_from_template(path_template: str, env_id: int) -> str: + """Resolve common env wildcard/template spellings to a concrete env path.""" + path = path_template + if "%d" in path: + return path % env_id + if "{}" in path: + return path.format(env_id) + path = path.replace("/World/envs/*", f"/World/envs/env_{env_id}") + path = path.replace("/World/envs/env_.*", f"/World/envs/env_{env_id}") + path = path.replace("/World/envs/env_.*/", f"/World/envs/env_{env_id}/") + return path + + +def _camera_concrete_paths(camera: Camera) -> list[str]: + view = getattr(camera, "_view", None) + prims = getattr(view, "prims", None) + if not prims: + return [] + return [prim.GetPath().pathString for prim in prims] + + +def find_camera_by_prim_path(camera_sensors: dict[str, Camera], cam_prim_path: str, env_indices: list[int]) -> Camera: + """Find a scene-owned Camera by config template or concrete camera prim paths.""" + wanted = {env_path_from_template(cam_prim_path, env_id) for env_id in env_indices} + for camera in camera_sensors.values(): + if getattr(camera.cfg, "prim_path", None) == cam_prim_path: + return camera + concrete = set(_camera_concrete_paths(camera)) + if wanted and wanted.issubset(concrete): + return camera + stage = sim_utils.get_current_stage() + stage_matches = [path for path in wanted if stage.GetPrimAtPath(path).IsValid()] if stage is not None else [] + if stage_matches: + raise RuntimeError( + f"cam_prim_path={cam_prim_path!r} matched USD camera prims, but no Isaac Lab Camera sensor owns them. " + "Add the camera to scene.sensors or leave tiled_cam_prim_path unset to use generated tiled cameras." + ) + raise RuntimeError(f"No Isaac Lab Camera sensor matched cam_prim_path={cam_prim_path!r}.") + + +def ensure_camera_initialized(camera: Camera) -> None: + """Initialize a visualizer-owned Camera created after the normal physics-ready callback.""" + if not camera.is_initialized: + camera._initialize_callback(None) + + +def create_visualizer_camera( + *, + num_envs: int, + camera_name: str = _GENERATED_CAMERA_NAME, + width: int, + height: int, + renderer_cfg: Any, +) -> tuple[Camera, list[str]]: + """Create an internal RGB Camera sensor for visualizer image views.""" + spawn = sim_utils.PinholeCameraCfg( + focal_length=24.0, + focus_distance=400.0, + horizontal_aperture=20.955, + clipping_range=(0.1, 1.0e5), + ) + generated_paths = [f"/World/envs/env_{env_id}/{camera_name}" for env_id in range(int(num_envs))] + for path in generated_paths: + if len(sim_utils.find_matching_prims(path)) == 0: + spawn.func(path, spawn, translation=(0.0, 0.0, 0.0), orientation=(0.0, 0.0, 0.0, 1.0)) + cfg = CameraCfg( + prim_path=f"/World/envs/env_.*/{camera_name}", + update_period=0.0, + height=int(height), + width=int(width), + data_types=["rgb"], + spawn=None, + renderer_cfg=renderer_cfg, + ) + camera = Camera(cfg) + ensure_camera_initialized(camera) + return camera, generated_paths + + +def remove_generated_prims(prim_paths: list[str] | None) -> None: + """Remove visualizer-owned camera prims from the current stage.""" + if not prim_paths: + return + stage = sim_utils.get_current_stage() + if stage is None: + return + for path in prim_paths: + prim = stage.GetPrimAtPath(path) + if prim.IsValid(): + stage.RemovePrim(path) + + +def camera_rgb_batch(camera: Camera, env_indices: list[int]) -> torch.Tensor: + """Return RGB output for selected env indices.""" + rgb = camera.data.output["rgb"] + if isinstance(rgb, wp.array): + rgb = wp.to_torch(rgb) + elif hasattr(rgb, "torch"): + rgb = rgb.torch + if env_indices: + index = torch.tensor(env_indices, dtype=torch.long, device=rgb.device) + return rgb.index_select(0, index) + return rgb + + +def compose_rgb_grid_tensor(rgb_batch: torch.Tensor) -> torch.Tensor: + """Compose an RGB batch into a near-square uint8 image grid without leaving its device.""" + if rgb_batch.ndim == 3: + return rgb_batch[..., :3].contiguous() + n, h, w, _ = rgb_batch.shape + cols = max(1, math.ceil(math.sqrt(n))) + rows = math.ceil(n / cols) + rgb = rgb_batch[..., :3] + pad = rows * cols - n + if pad > 0: + rgb = torch.cat([rgb, torch.zeros((pad, h, w, 3), dtype=rgb.dtype, device=rgb.device)], dim=0) + return rgb.reshape(rows, cols, h, w, 3).permute(0, 2, 1, 3, 4).reshape(rows * h, cols * w, 3).contiguous() + + +def compute_tile_resolution(window_width: int, window_height: int, num_tiles: int) -> tuple[int, int]: + """Derive a conservative per-tile resolution from the visualizer window.""" + if window_width <= 0 or window_height <= 0: + raise ValueError(f"Window dimensions must be positive, got {window_width}x{window_height}.") + cols = max(1, math.ceil(math.sqrt(max(1, num_tiles)))) + rows = math.ceil(max(1, num_tiles) / cols) + return max(1, int(window_width) // cols), max(1, int(window_height) // rows) + + +def _normalize_env0_path(path_template: str) -> str: + """Resolve env template spellings to env_0 for path comparison.""" + return env_path_from_template(path_template, 0) + + +def _scene_articulation_positions(scene: Any, prim_path_template: str, env_indices: list[int]) -> torch.Tensor | None: + """Resolve follow positions from scene articulation state when the path targets an asset/body.""" + follow_env0 = _normalize_env0_path(prim_path_template) + for asset in getattr(scene, "articulations", {}).values(): + asset_path = _normalize_env0_path(getattr(asset.cfg, "prim_path", "")) + if not asset_path: + continue + if follow_env0 == asset_path: + return asset.data.root_pos_w.torch[env_indices].detach().cpu() + prefix = asset_path + "/" + if not follow_env0.startswith(prefix): + continue + body_name = follow_env0.removeprefix(prefix).split("/")[-1] + if body_name not in asset.body_names: + continue + body_ids, _ = asset.find_bodies(body_name) + if not body_ids: + continue + return asset.data.body_pos_w.torch[env_indices, int(body_ids[0])].detach().cpu() + return None + + +def prim_world_positions( + stage: Any, prim_path_template: str, env_indices: list[int], scene: Any | None = None +) -> torch.Tensor: + """Return world-space translations for concrete prim paths resolved from env ids. + + Uses ``FrameView`` first so PhysX/Fabric-backed transforms are current; falls + back to USD only if the backend view cannot be constructed. + """ + from pxr import UsdGeom + + from isaaclab.sim.views import FrameView + + if scene is not None: + positions = _scene_articulation_positions(scene, prim_path_template, env_indices) + if positions is not None: + return positions + + xform_cache = UsdGeom.XformCache() + positions = [] + for env_id in env_indices: + prim_path = env_path_from_template(prim_path_template, env_id) + try: + view = FrameView(prim_path, device="cpu", stage=stage) + if view.count != 1: + raise RuntimeError(f"expected one prim, got {view.count}") + pos_w, _ = view.get_world_poses() + pos = pos_w.torch[0].detach().cpu() + positions.append((float(pos[0]), float(pos[1]), float(pos[2]))) + except Exception: + prim = stage.GetPrimAtPath(prim_path) + if not prim.IsValid(): + raise RuntimeError(f"tiled_cam_target_prim_path resolved to missing prim: {prim_path!r}.") + transform = xform_cache.GetLocalToWorldTransform(prim) + translation = transform.ExtractTranslation() + positions.append((float(translation[0]), float(translation[1]), float(translation[2]))) + return torch.tensor(positions, dtype=torch.float32) + + +def apply_camera_view_from_origins( + camera: Camera, + origins: torch.Tensor, + eye: tuple[float, float, float], + lookat: tuple[float, float, float], + env_ids: list[int] | None = None, +) -> None: + """Set camera poses from origins plus relative eye/lookat offsets.""" + device = camera.device + origins = origins.to(device=device) + eye_offset = torch.tensor(eye, dtype=torch.float32, device=device).unsqueeze(0) + lookat_offset = torch.tensor(lookat, dtype=torch.float32, device=device).unsqueeze(0) + camera.set_world_poses_from_view(origins + eye_offset, origins + lookat_offset, env_ids=env_ids) + camera._update_poses(None) + + +def apply_camera_target_positions( + camera: Camera, + target_positions: torch.Tensor, + eye: tuple[float, float, float], + env_ids: list[int] | None = None, +) -> tuple[torch.Tensor, torch.Tensor]: + """Set generated tiled camera poses as target-relative eye offsets.""" + device = camera.device + target_positions = target_positions.to(device=device) + eye_offset = torch.tensor(eye, dtype=torch.float32, device=device).unsqueeze(0) + eyes = target_positions + eye_offset + targets = target_positions + camera.set_world_poses_from_view(eyes, targets, env_ids=env_ids) + camera._update_poses(None) + return eyes.detach().cpu(), targets.detach().cpu() diff --git a/source/isaaclab/isaaclab/markers/visualization_markers.py b/source/isaaclab/isaaclab/markers/visualization_markers.py index 2e418bbecad..667f9b4ddf7 100644 --- a/source/isaaclab/isaaclab/markers/visualization_markers.py +++ b/source/isaaclab/isaaclab/markers/visualization_markers.py @@ -250,7 +250,10 @@ def _ensure_backends_initialized(self) -> None: self._ensure_kit_backend() return - if any(viz.supports_markers() and viz.pumps_app_update() and viz.cfg.enable_markers for viz in sim.visualizers): + needs_kit_backend = sim.is_rendering or any( + viz.supports_markers() and viz.pumps_app_update() and viz.cfg.enable_markers for viz in sim.visualizers + ) + if needs_kit_backend: self._ensure_kit_backend() if any( viz.supports_markers() and not viz.pumps_app_update() and viz.cfg.enable_markers for viz in sim.visualizers diff --git a/source/isaaclab/isaaclab/physics/__init__.pyi b/source/isaaclab/isaaclab/physics/__init__.pyi index d2ceced82eb..b4f91086621 100644 --- a/source/isaaclab/isaaclab/physics/__init__.pyi +++ b/source/isaaclab/isaaclab/physics/__init__.pyi @@ -8,10 +8,7 @@ __all__ = [ "PhysicsEvent", "PhysicsManager", "PhysicsCfg", - "SceneDataBackend", - "SceneDataFormat", ] from .physics_manager import CallbackHandle, PhysicsEvent, PhysicsManager from .physics_manager_cfg import PhysicsCfg -from .scene_data_backend import SceneDataBackend, SceneDataFormat diff --git a/source/isaaclab/isaaclab/physics/physics_manager.py b/source/isaaclab/isaaclab/physics/physics_manager.py index e6abd828713..658679d5db0 100644 --- a/source/isaaclab/isaaclab/physics/physics_manager.py +++ b/source/isaaclab/isaaclab/physics/physics_manager.py @@ -15,7 +15,7 @@ from typing import TYPE_CHECKING, Any, ClassVar if TYPE_CHECKING: - from isaaclab.physics.scene_data_backend import SceneDataBackend + from isaaclab.scene_data import SceneDataBackend from isaaclab.sim.simulation_context import SimulationContext logger = logging.getLogger(__name__) diff --git a/source/isaaclab/isaaclab/physics/scene_data_backend.py b/source/isaaclab/isaaclab/physics/scene_data_backend.py deleted file mode 100644 index d94bea9e76c..00000000000 --- a/source/isaaclab/isaaclab/physics/scene_data_backend.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Backend interface and data formats for the scene data provider. - -These types live in :mod:`isaaclab.physics` rather than -:mod:`isaaclab.scene.scene_data_provider` so that physics backends -(``isaaclab_physx``, ``isaaclab_newton``) can subclass -:class:`SceneDataBackend` without pulling :mod:`isaaclab.scene` into the -``AppLauncher`` pre-launch import chain. ``AppLauncher._create_app`` pops -``*lab*`` modules from ``sys.modules`` during Kit init and any submodule -imported during that window ends up orphaned from its parent's -``__dict__`` after restoration. -""" - -from __future__ import annotations - -import warp as wp - - -class SceneDataFormat: - @wp.struct - class Vec3_Quat: - positions: wp.array(dtype=wp.vec3f) = None - orientations: wp.array(dtype=wp.quatf) = None - - @wp.struct - class Vec3_Matrix33: - positions: wp.array(dtype=wp.vec3f) = None - orientations: wp.array(dtype=wp.mat33f) = None - - @wp.struct - class Transform: - transforms: wp.array(dtype=wp.transformf) = None - - @wp.struct - class Matrix44: - matrices: wp.array(dtype=wp.mat44f) = None - - -class SceneDataBackend: - @property - def transforms( - self, - ) -> ( - SceneDataFormat.Vec3_Quat | SceneDataFormat.Transform | SceneDataFormat.Matrix44 | SceneDataFormat.Vec3_Matrix33 - ): - """Return the sim backends transforms as one of the SceneDataFormat structs.""" - raise NotImplementedError - - @property - def transform_count(self) -> int: - """Return the number of transforms in the sim backend.""" - raise NotImplementedError - - @property - def transform_paths(self) -> list[str]: - """Return the paths for each transform.""" - raise NotImplementedError diff --git a/source/isaaclab/isaaclab/scene/interactive_scene.py b/source/isaaclab/isaaclab/scene/interactive_scene.py index 70756007d6f..b8df495d1e7 100644 --- a/source/isaaclab/isaaclab/scene/interactive_scene.py +++ b/source/isaaclab/isaaclab/scene/interactive_scene.py @@ -10,7 +10,7 @@ from typing import TYPE_CHECKING, Any if TYPE_CHECKING: - from isaaclab_physx.assets import DeformableObject, SurfaceGripper + from isaaclab_physx.assets import SurfaceGripper from isaaclab.renderers.base_renderer import BaseRenderer @@ -25,6 +25,8 @@ Articulation, ArticulationCfg, AssetBaseCfg, + DeformableObject, + DeformableObjectCfg, RigidObject, RigidObjectCfg, RigidObjectCollection, @@ -878,7 +880,7 @@ def _is_scene_setup_from_cfg(self) -> bool: def _add_entities_from_cfg(self): # noqa: C901 """Add scene entities from the config.""" - from isaaclab_physx.assets import DeformableObjectCfg, SurfaceGripperCfg # noqa: PLC0415 + from isaaclab_physx.assets import SurfaceGripperCfg # noqa: PLC0415 # store paths that are in global collision filter self._global_prim_paths = list() diff --git a/source/isaaclab/isaaclab/scene_data/__init__.py b/source/isaaclab/isaaclab/scene_data/__init__.py new file mode 100644 index 00000000000..9e3eaa88579 --- /dev/null +++ b/source/isaaclab/isaaclab/scene_data/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Sub-package containing the scene data provider and backend interface. + +The :class:`SceneDataProvider` bridges physics simulation backends and the +consumers that read scene transforms (renderers and visualizers). Physics +backends implement :class:`SceneDataBackend` to expose their current +transforms in one of the :class:`SceneDataFormat` Warp struct variants; +the provider converts and remaps them on demand for each consumer. + +This package is deliberately separate from :mod:`isaaclab.scene` so that +physics backends (``isaaclab_physx``, ``isaaclab_newton``) can subclass +:class:`SceneDataBackend` without pulling :mod:`isaaclab.scene` into the +``AppLauncher`` pre-launch import chain. +""" + +from isaaclab.utils.module import lazy_export + +lazy_export() diff --git a/source/isaaclab/isaaclab/scene_data/__init__.pyi b/source/isaaclab/isaaclab/scene_data/__init__.pyi new file mode 100644 index 00000000000..2e17d33cf19 --- /dev/null +++ b/source/isaaclab/isaaclab/scene_data/__init__.pyi @@ -0,0 +1,13 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +__all__ = [ + "SceneDataBackend", + "SceneDataFormat", + "SceneDataProvider", +] + +from .scene_data_backend import SceneDataBackend, SceneDataFormat +from .scene_data_provider import SceneDataProvider diff --git a/source/isaaclab/isaaclab/scene_data/scene_data_backend.py b/source/isaaclab/isaaclab/scene_data/scene_data_backend.py new file mode 100644 index 00000000000..183b0824684 --- /dev/null +++ b/source/isaaclab/isaaclab/scene_data/scene_data_backend.py @@ -0,0 +1,91 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Backend interface and data formats for the scene data provider. + +These types live in :mod:`isaaclab.scene_data` rather than +:mod:`isaaclab.scene` so that physics backends (``isaaclab_physx``, +``isaaclab_newton``) can subclass :class:`SceneDataBackend` without pulling +:mod:`isaaclab.scene` into the ``AppLauncher`` pre-launch import chain. +``AppLauncher._create_app`` pops ``*lab*`` modules from ``sys.modules`` +during Kit init and any submodule imported during that window ends up +orphaned from its parent's ``__dict__`` after restoration. +""" + +from __future__ import annotations + +import warp as wp + +# Under Sphinx ``autodoc_mock_imports``, ``wp.struct`` is a ``_MockObject`` +# that replaces the decorated class with another mock, hiding its docstring +# and fields from autodoc. Fall back to an identity decorator when warp is +# mocked so the documentation builds from the source classes directly. +if getattr(wp, "__sphinx_mock__", False): + + def wp_struct(cls): + return cls +else: + wp_struct = wp.struct + + +class SceneDataFormat: + """Warp struct variants describing the transform layouts that a + :class:`SceneDataBackend` may publish to consumers. + """ + + @wp_struct + class Vec3_Quat: + """Separate position and quaternion arrays.""" + + positions: wp.array(dtype=wp.vec3f) = None + """Per-transform positions [m].""" + + orientations: wp.array(dtype=wp.quatf) = None + """Per-transform orientations as quaternions.""" + + @wp_struct + class Vec3_Matrix33: + """Separate position and rotation-matrix arrays.""" + + positions: wp.array(dtype=wp.vec3f) = None + """Per-transform positions [m].""" + + orientations: wp.array(dtype=wp.mat33f) = None + """Per-transform orientations as 3x3 rotation matrices.""" + + @wp_struct + class Transform: + """Packed warp transforms (position + quaternion).""" + + transforms: wp.array(dtype=wp.transformf) = None + """Per-transform packed position + orientation transforms [m, -].""" + + @wp_struct + class Matrix44: + """Packed 4x4 homogeneous transform matrices.""" + + matrices: wp.array(dtype=wp.mat44f) = None + """Per-transform 4x4 homogeneous transform matrices [m].""" + + +class SceneDataBackend: + @property + def transforms( + self, + ) -> ( + SceneDataFormat.Vec3_Quat | SceneDataFormat.Transform | SceneDataFormat.Matrix44 | SceneDataFormat.Vec3_Matrix33 + ): + """Return the sim backends transforms as one of the SceneDataFormat structs.""" + raise NotImplementedError + + @property + def transform_count(self) -> int: + """Return the number of transforms in the sim backend.""" + raise NotImplementedError + + @property + def transform_paths(self) -> list[str]: + """Return the paths for each transform.""" + raise NotImplementedError diff --git a/source/isaaclab/isaaclab/scene/scene_data_provider.py b/source/isaaclab/isaaclab/scene_data/scene_data_provider.py similarity index 94% rename from source/isaaclab/isaaclab/scene/scene_data_provider.py rename to source/isaaclab/isaaclab/scene_data/scene_data_provider.py index a44700dd575..e31e9d9da28 100644 --- a/source/isaaclab/isaaclab/scene/scene_data_provider.py +++ b/source/isaaclab/isaaclab/scene_data/scene_data_provider.py @@ -13,7 +13,7 @@ import numpy as np import warp as wp -from isaaclab.physics.scene_data_backend import SceneDataBackend, SceneDataFormat +from .scene_data_backend import SceneDataBackend, SceneDataFormat if TYPE_CHECKING: from pxr import Usd @@ -28,6 +28,29 @@ def __init__(self, backend: SceneDataBackend): """ self.backend = backend self._num_envs_cache: int | None = None + self._interactive_scene: Any | None = None + + def set_interactive_scene(self, scene: Any) -> None: + """Attach the active interactive scene for scene-owned sensor discovery.""" + self._interactive_scene = scene + + def get_interactive_scene(self) -> Any | None: + """Return the registered interactive scene, if available.""" + return self._interactive_scene + + def get_camera_sensors(self) -> dict[str, Any]: + """Return Isaac Lab camera sensors keyed by scene sensor name.""" + if self._interactive_scene is None: + return {} + try: + from isaaclab.sensors.camera import Camera + except ImportError: + return {} + return { + name: sensor + for name, sensor in getattr(self._interactive_scene, "sensors", {}).items() + if isinstance(sensor, Camera) + } @property def transform_count(self) -> int: @@ -56,6 +79,10 @@ def usd_stage(self) -> Usd.Stage | None: except Exception: return None + def get_usd_stage(self) -> Usd.Stage | None: + """Return the USD stage for callers using the older method-style API.""" + return self.usd_stage + @property def num_envs(self) -> int: """Number of environments discovered from ``/World/envs/env_`` prims. diff --git a/source/isaaclab/isaaclab/sensors/camera/camera.py b/source/isaaclab/isaaclab/sensors/camera/camera.py index 588d4cb000f..33c5e3c3fc2 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera.py @@ -330,7 +330,7 @@ def set_world_poses( elif not isinstance(positions, torch.Tensor): positions = torch.tensor(positions, device=self._device) positions = positions.to(device=self._device, dtype=torch.float32).reshape(-1, 3) - pos_wp = wp.from_torch(positions.contiguous()) + pos_wp = wp.from_torch(positions.contiguous(), dtype=wp.vec3f) ori_wp = None if orientations is not None: if isinstance(orientations, np.ndarray): @@ -339,7 +339,7 @@ def set_world_poses( orientations = torch.tensor(orientations, device=self._device) orientations = orientations.to(device=self._device, dtype=torch.float32).reshape(-1, 4) orientations = convert_camera_frame_orientation_convention(orientations, origin=convention, target="opengl") - ori_wp = wp.from_torch(orientations.contiguous()) + ori_wp = wp.from_torch(orientations.contiguous(), dtype=wp.vec4f) idx_wp = self._resolve_env_ids_wp(env_ids) self._view.set_world_poses(pos_wp, ori_wp, idx_wp) @@ -400,8 +400,8 @@ def set_world_poses_from_view( orientations = quat_from_matrix(rotation_matrix) idx_wp = wp.from_torch(env_ids_torch.contiguous(), dtype=wp.int32) self._view.set_world_poses( - wp.from_torch(eyes.contiguous()), - wp.from_torch(orientations.contiguous()), + wp.from_torch(eyes.contiguous(), dtype=wp.vec3f), + wp.from_torch(orientations.contiguous(), dtype=wp.vec4f), idx_wp, ) diff --git a/source/isaaclab/isaaclab/sim/__init__.py b/source/isaaclab/isaaclab/sim/__init__.py index e1458e7873f..a0f1d706e1e 100644 --- a/source/isaaclab/isaaclab/sim/__init__.py +++ b/source/isaaclab/isaaclab/sim/__init__.py @@ -40,8 +40,9 @@ "PhysxJointDrivePropertiesCfg", "CollisionPropertiesCfg", "PhysxCollisionPropertiesCfg", - "PhysXCollisionPropertiesCfg", + "DeformableBodyPropertiesCfg", "PhysxDeformableCollisionPropertiesCfg", + "PhysxDeformableBodyPropertiesCfg", "ArticulationRootPropertiesCfg", "PhysxArticulationRootPropertiesCfg", "MeshCollisionPropertiesCfg", @@ -63,8 +64,12 @@ # Names that moved out of this package into ``isaaclab_physx.sim.spawners.materials``. _PHYSX_FORWARDS_MATERIALS = frozenset({ + "DeformableBodyMaterialCfg", "RigidBodyMaterialCfg", + "SurfaceDeformableBodyMaterialCfg", "PhysxRigidBodyMaterialCfg", + "PhysxDeformableBodyMaterialCfg", + "PhysxSurfaceDeformableBodyMaterialCfg", }) _PHYSX_FORWARDS = _PHYSX_FORWARDS_SCHEMAS | _PHYSX_FORWARDS_MATERIALS diff --git a/source/isaaclab/isaaclab/sim/__init__.pyi b/source/isaaclab/isaaclab/sim/__init__.pyi index 2d5edfdf921..0c787cc64c6 100644 --- a/source/isaaclab/isaaclab/sim/__init__.pyi +++ b/source/isaaclab/isaaclab/sim/__init__.pyi @@ -22,11 +22,13 @@ __all__ = [ "activate_contact_sensors", "define_articulation_root_properties", "define_collision_properties", + "define_deformable_body_properties", "define_mass_properties", "define_mesh_collision_properties", "define_rigid_body_properties", "modify_articulation_root_properties", "modify_collision_properties", + "modify_deformable_body_properties", "modify_fixed_tendon_properties", "modify_joint_drive_properties", "modify_mass_properties", @@ -39,6 +41,8 @@ __all__ = [ "CollisionBaseCfg", "ConvexDecompositionPropertiesCfg", "ConvexHullPropertiesCfg", + "DeformableBodyPropertiesBaseCfg", + "DeformableBodyPropertiesCfg", "FixedTendonPropertiesCfg", "JointDriveBaseCfg", "MassPropertiesCfg", @@ -60,6 +64,7 @@ __all__ = [ "TriangleMeshSimplificationPropertiesCfg", "SpawnerCfg", "RigidObjectSpawnerCfg", + "DeformableObjectSpawnerCfg", "spawn_from_mjcf", "spawn_from_urdf", "spawn_from_usd", @@ -78,8 +83,13 @@ __all__ = [ "LightCfg", "SphereLightCfg", "spawn_rigid_body_material", + "spawn_deformable_body_material", "PhysicsMaterialCfg", "RigidBodyMaterialCfg", + "DeformableBodyMaterialBaseCfg", + "DeformableBodyMaterialCfg", + "SurfaceDeformableBodyMaterialBaseCfg", + "SurfaceDeformableBodyMaterialCfg", "spawn_from_mdl_file", "spawn_preview_surface", "GlassMdlCfg", @@ -90,15 +100,15 @@ __all__ = [ "spawn_mesh_cone", "spawn_mesh_cuboid", "spawn_mesh_cylinder", + "spawn_mesh_rectangle", "spawn_mesh_sphere", - "spawn_mesh_square", "MeshCapsuleCfg", "MeshCfg", "MeshConeCfg", "MeshCuboidCfg", "MeshCylinderCfg", + "MeshRectangleCfg", "MeshSphereCfg", - "MeshSquareCfg", "spawn_camera", "spawn_sensor_frame", "FisheyeCameraCfg", @@ -198,11 +208,13 @@ from .schemas import ( activate_contact_sensors, define_articulation_root_properties, define_collision_properties, + define_deformable_body_properties, define_mass_properties, define_mesh_collision_properties, define_rigid_body_properties, modify_articulation_root_properties, modify_collision_properties, + modify_deformable_body_properties, modify_fixed_tendon_properties, modify_joint_drive_properties, modify_mass_properties, @@ -215,6 +227,8 @@ from .schemas import ( CollisionBaseCfg, ConvexDecompositionPropertiesCfg, ConvexHullPropertiesCfg, + DeformableBodyPropertiesBaseCfg, + DeformableBodyPropertiesCfg, FixedTendonPropertiesCfg, JointDriveBaseCfg, MassPropertiesCfg, @@ -240,6 +254,7 @@ NewtonRigidBodyPropertiesCfg = ... from .spawners import ( SpawnerCfg, RigidObjectSpawnerCfg, + DeformableObjectSpawnerCfg, spawn_from_mjcf, spawn_from_urdf, spawn_from_usd, @@ -258,8 +273,13 @@ from .spawners import ( LightCfg, SphereLightCfg, spawn_rigid_body_material, + spawn_deformable_body_material, PhysicsMaterialCfg, RigidBodyMaterialCfg, + DeformableBodyMaterialBaseCfg, + DeformableBodyMaterialCfg, + SurfaceDeformableBodyMaterialBaseCfg, + SurfaceDeformableBodyMaterialCfg, spawn_from_mdl_file, spawn_preview_surface, GlassMdlCfg, @@ -270,15 +290,15 @@ from .spawners import ( spawn_mesh_cone, spawn_mesh_cuboid, spawn_mesh_cylinder, + spawn_mesh_rectangle, spawn_mesh_sphere, - spawn_mesh_square, MeshCapsuleCfg, MeshCfg, MeshConeCfg, MeshCuboidCfg, MeshCylinderCfg, + MeshRectangleCfg, MeshSphereCfg, - MeshSquareCfg, spawn_camera, spawn_sensor_frame, FisheyeCameraCfg, diff --git a/source/isaaclab/isaaclab/sim/schemas/__init__.py b/source/isaaclab/isaaclab/sim/schemas/__init__.py index 223627d4e52..ed0e5a30ef0 100644 --- a/source/isaaclab/isaaclab/sim/schemas/__init__.py +++ b/source/isaaclab/isaaclab/sim/schemas/__init__.py @@ -46,8 +46,9 @@ "PhysxJointDrivePropertiesCfg", "CollisionPropertiesCfg", "PhysxCollisionPropertiesCfg", - "PhysXCollisionPropertiesCfg", + "DeformableBodyPropertiesCfg", "PhysxDeformableCollisionPropertiesCfg", + "PhysxDeformableBodyPropertiesCfg", "ArticulationRootPropertiesCfg", "PhysxArticulationRootPropertiesCfg", "MeshCollisionPropertiesCfg", diff --git a/source/isaaclab/isaaclab/sim/schemas/__init__.pyi b/source/isaaclab/isaaclab/sim/schemas/__init__.pyi index a9306afdf41..8d753b553d6 100644 --- a/source/isaaclab/isaaclab/sim/schemas/__init__.pyi +++ b/source/isaaclab/isaaclab/sim/schemas/__init__.pyi @@ -11,11 +11,13 @@ __all__ = [ "define_actuator_properties", "define_articulation_root_properties", "define_collision_properties", + "define_deformable_body_properties", "define_mass_properties", "define_mesh_collision_properties", "define_rigid_body_properties", "modify_articulation_root_properties", "modify_collision_properties", + "modify_deformable_body_properties", "modify_fixed_tendon_properties", "modify_joint_drive_properties", "modify_mass_properties", @@ -26,6 +28,8 @@ __all__ = [ "BoundingCubePropertiesCfg", "BoundingSpherePropertiesCfg", "CollisionBaseCfg", + "DeformableBodyPropertiesBaseCfg", + "DeformableBodyPropertiesCfg", "JointDriveBaseCfg", "MassPropertiesCfg", "MeshCollisionBaseCfg", @@ -47,11 +51,13 @@ from .schemas import ( activate_contact_sensors, define_articulation_root_properties, define_collision_properties, + define_deformable_body_properties, define_mass_properties, define_mesh_collision_properties, define_rigid_body_properties, modify_articulation_root_properties, modify_collision_properties, + modify_deformable_body_properties, modify_fixed_tendon_properties, modify_joint_drive_properties, modify_mass_properties, @@ -67,6 +73,8 @@ from .schemas_cfg import ( BoundingCubePropertiesCfg, BoundingSpherePropertiesCfg, CollisionBaseCfg, + DeformableBodyPropertiesBaseCfg, + DeformableBodyPropertiesCfg, JointDriveBaseCfg, MassPropertiesCfg, MeshCollisionBaseCfg, diff --git a/source/isaaclab/isaaclab/sim/schemas/schemas.py b/source/isaaclab/isaaclab/sim/schemas/schemas.py index b7a74e27ff2..8bd2c314bf9 100644 --- a/source/isaaclab/isaaclab/sim/schemas/schemas.py +++ b/source/isaaclab/isaaclab/sim/schemas/schemas.py @@ -10,14 +10,19 @@ import logging import math -from pxr import Usd, UsdPhysics +import numpy as np +import warp as wp + +from pxr import Sdf, Usd, UsdGeom, UsdPhysics from isaaclab.sim.utils.stage import get_current_stage from isaaclab.utils.string import to_camel_case from ..utils import ( apply_nested, + create_prim, find_global_fixed_joint_prim, + get_all_matching_child_prims, safe_set_attribute_on_usd_prim, safe_set_attribute_on_usd_schema, ) @@ -1071,3 +1076,360 @@ def modify_mesh_collision_properties( # success return True + + +""" +Deformable body properties. +""" + + +@wp.kernel +def _fix_tet_winding_kernel( + points: wp.array(dtype=wp.vec3), + tet_indices: wp.array(ndim=2, dtype=wp.int32), +): + """Flip any tet with negative signed volume by swapping its last two vertex indices. + + ``UsdGeom.TetMesh`` and :meth:`UsdGeom.TetMesh.ComputeSurfaceFaces` require a + right-handed tet winding (positive signed volume). Swapping indices 2 and 3 + reverses the orientation without changing which four vertices form the tet. + """ + i = wp.tid() + v0 = tet_indices[i, 0] + v1 = tet_indices[i, 1] + v2 = tet_indices[i, 2] + v3 = tet_indices[i, 3] + p0 = points[v0] + e1 = points[v1] - p0 + e2 = points[v2] - p0 + e3 = points[v3] - p0 + signed_volume = wp.dot(e1, wp.cross(e2, e3)) + if signed_volume < 0.0: + tet_indices[i, 2] = v3 + tet_indices[i, 3] = v2 + + +def define_deformable_body_properties( + prim_path: str, + cfg: schemas_cfg.DeformableBodyPropertiesBaseCfg, + stage: Usd.Stage | None = None, + deformable_type: str = "volume", + sim_mesh_prim_path: str | None = None, +): + """Apply the deformable body schema on the input prim and set its properties. The input prim should + have a visual surface mesh as child. Volume deformables will have their simulation tetrahedral mesh + automatically computed from the surface mesh of the input prim. Surface deformables simply copy the visual mesh + as simulation mesh. + + See :func:`modify_deformable_body_properties` for more details on how the properties are set. + + .. note:: + If the input prim is not a mesh, this function will traverse the prim and find the first mesh + under it. If no mesh or multiple meshes are found, an error is raised. This is because the deformable + body schema can only be applied to a single mesh. + + Args: + prim_path: The prim path where to apply the deformable body schema. + cfg: The configuration for the deformable body. + stage: The stage where to find the prim. Defaults to None, in which case the + current stage is used. + deformable_type: The type of the deformable body (surface or volume). + This is used to determine which USD API to use for the deformable body. Defaults to "volume". + sim_mesh_prim_path: Optional override for the simulation mesh creation prim path. + Ignored when pre-tetrahedralized mesh is found for volume deformables. + If None, it is set to ``{prim_path}/sim_mesh``. + + Raises: + ValueError: When the prim path is not valid. + ValueError: When the prim has no mesh or multiple meshes. + RuntimeError: When setting the deformable body properties fails. + """ + from omni.physx.scripts import deformableUtils + + # get stage handle + if stage is None: + stage = get_current_stage() + + # get USD prim + root_prim = stage.GetPrimAtPath(prim_path) + # check if prim path is valid + if not root_prim.IsValid(): + raise ValueError(f"Prim path '{prim_path}' is not valid.") + + sim_mesh_prim = None + # for volume deformables, we check if a pre-tetrahedralized TetMesh exists for the sim_mesh + if deformable_type == "volume": + matching_prims = get_all_matching_child_prims(prim_path, lambda p: p.GetTypeName() == "TetMesh") + if len(matching_prims) == 0: + sim_mesh_prim = None + elif len(matching_prims) > 1: + # get list of all meshes found + mesh_paths = [p.GetPrimPath() for p in matching_prims] + raise ValueError( + f"Found multiple tetrahedral meshes in '{prim_path}': {mesh_paths}." + " Deformable body schema can only be applied to one mesh for now." + ) + else: + # found existing tetmesh + sim_mesh_prim = matching_prims[0] + if not sim_mesh_prim.IsValid(): + raise ValueError(f"Mesh prim path '{sim_mesh_prim.GetPrimPath()}' is not valid.") + + # Search for a visual surface mesh for both surface and volume deformables + matching_prims = get_all_matching_child_prims(prim_path, lambda p: p.GetTypeName() == "Mesh") + # check if the visual surface mesh is valid + if len(matching_prims) == 0: + # in case a TetMesh is found but no Mesh is found, we use the TetMesh surface as visual. + if sim_mesh_prim is not None: + tet_mesh_prim = UsdGeom.TetMesh(sim_mesh_prim) + surface_indices = UsdGeom.TetMesh.ComputeSurfaceFaces(tet_mesh_prim, Usd.TimeCode.Default()) + if surface_indices is None or len(surface_indices) == 0: + raise ValueError( + f"Deformable body at '{prim_path}' has no surface indices on its TetMesh prim; " + "cannot sync to visual mesh." + ) + # create visual mesh + vis_mesh_prim = create_prim( + prim_path + "/vis_mesh", + prim_type="Mesh", + attributes={ + "points": tet_mesh_prim.GetPointsAttr().Get(), + "faceVertexIndices": np.asarray(surface_indices).flatten(), + "faceVertexCounts": [3] * len(surface_indices), + }, + stage=stage, + ) + matching_prims = [vis_mesh_prim] + else: + raise ValueError(f"Could not find any visual mesh in '{prim_path}'. Please check asset.") + if len(matching_prims) > 1: + # get list of all meshes found + mesh_paths = [p.GetPrimPath() for p in matching_prims] + raise ValueError( + f"Found multiple visual meshes in '{prim_path}': {mesh_paths}." + " Deformable body schema can only be applied to one mesh for now." + ) + vis_mesh_prim = matching_prims[0] + + # check if the prim is valid + if not vis_mesh_prim.IsValid(): + raise ValueError(f"Mesh prim path '{vis_mesh_prim.GetPrimPath()}' is not valid.") + + # remove potential previous configuration + deformableUtils.remove_deformable_body(stage, prim_path) + + # create and set simulation/root prim properties based on the type of the deformable mesh (surface vs volume) + sim_mesh_prim_path = prim_path + "/sim_mesh" if sim_mesh_prim_path is None else sim_mesh_prim_path + # extract visual surface mesh vertices and faces + vertices = np.array(vis_mesh_prim.GetAttribute("points").Get()) + faces = np.array(vis_mesh_prim.GetAttribute("faceVertexIndices").Get()).flatten() + face_counts = np.array(vis_mesh_prim.GetAttribute("faceVertexCounts").Get()) + if deformable_type == "surface": + # create simulation mesh as copy of visual mesh + sim_mesh_prim = create_prim( + sim_mesh_prim_path, + prim_type="Mesh", + attributes={ + "points": vertices, + "faceVertexIndices": faces, + "faceVertexCounts": face_counts, + }, + stage=stage, + ) + + # apply sim API + if not sim_mesh_prim.ApplyAPI("OmniPhysicsSurfaceDeformableSimAPI"): + raise RuntimeError(f"Failed to set surface deformable body API on prim '{sim_mesh_prim_path}'.") + + # apply collision API + if not sim_mesh_prim.ApplyAPI(UsdPhysics.CollisionAPI): + raise RuntimeError(f"Failed to set surface deformable collision API on prim '{sim_mesh_prim_path}'.") + + # set rest-shape attributes required by OmniPhysicsSurfaceDeformableSimAPI + sim_mesh_prim.GetAttribute("omniphysics:restShapePoints").Set(vertices) + sim_mesh_prim.GetAttribute("omniphysics:restTriVtxIndices").Set(faces) + + elif deformable_type == "volume": + if sim_mesh_prim is None: + try: + from pytetwild import tetrahedralize + except ImportError as exc: + raise ImportError( + "Automatic tetrahedralization of volume deformables requires the optional 'pytetwild' " + "package. Install pytetwild or provide a pre-tetrahedralized UsdGeom.TetMesh under the " + f"deformable prim '{prim_path}'." + ) from exc + + tet_mesh_points, tet_mesh_indices = tetrahedralize( + vertices, + faces.reshape(-1, 3), + edge_length_fac=0.1, + simplify=False, + epsilon=1e-2, + coarsen=True, + ) + # pytetwild's default ordering does not guarantee positive signed volume, which + # ``UsdGeom.TetMesh`` and ``ComputeSurfaceFaces`` require. Flip any inverted tets. + device = "cpu" + _tet_points_wp = wp.array(tet_mesh_points.astype(np.float32), dtype=wp.vec3, device=device) + _tet_indices_wp = wp.array( + np.asarray(tet_mesh_indices, dtype=np.int32).reshape(-1, 4), dtype=wp.int32, device=device + ) + wp.launch( + _fix_tet_winding_kernel, + dim=_tet_indices_wp.shape[0], + inputs=[_tet_points_wp, _tet_indices_wp], + device=device, + ) + tet_mesh_indices = _tet_indices_wp.numpy() + sim_mesh_prim = create_prim( + sim_mesh_prim_path, + prim_type="TetMesh", + attributes={ + "points": tet_mesh_points, + "tetVertexIndices": tet_mesh_indices, + }, + stage=stage, + ) + + # apply sim API + if not sim_mesh_prim.ApplyAPI("OmniPhysicsVolumeDeformableSimAPI"): + raise RuntimeError(f"Failed to set volume deformable body API on prim '{sim_mesh_prim_path}'.") + + # apply collision API + if not sim_mesh_prim.ApplyAPI(UsdPhysics.CollisionAPI): + raise RuntimeError(f"Failed to set volume deformable collision API on prim '{sim_mesh_prim_path}'.") + + # set surface faces and rest-shape attributes required by OmniPhysicsVolumeDeformableSimAPI + surface_face_indices = UsdGeom.TetMesh.ComputeSurfaceFaces( + UsdGeom.TetMesh(sim_mesh_prim), Usd.TimeCode.Default() + ) + UsdGeom.TetMesh(sim_mesh_prim).GetSurfaceFaceVertexIndicesAttr().Set(surface_face_indices) + sim_mesh_prim.GetAttribute("omniphysics:restShapePoints").Set(sim_mesh_prim.GetAttribute("points").Get()) + sim_mesh_prim.GetAttribute("omniphysics:restTetVtxIndices").Set( + sim_mesh_prim.GetAttribute("tetVertexIndices").Get() + ) + + else: + raise ValueError( + f"""Unsupported deformable type: '{deformable_type}'. + Only surface and volume deformables are supported.""" + ) + + # TODO: Temporary solution: Overwrite visual mesh with tet mesh surface points or copy + # surface sim mesh to vis mesh. In the future we can have separate visual from simulation mesh. + # This currently does not work if an asset is loaded where the visual mesh is not the simulation mesh surface. + vis_mesh = UsdGeom.Mesh(vis_mesh_prim) + if deformable_type == "volume": + tet_mesh_prim = UsdGeom.TetMesh(sim_mesh_prim) + surface_indices = tet_mesh_prim.GetSurfaceFaceVertexIndicesAttr().Get() + if surface_indices is None or len(surface_indices) == 0: + raise ValueError( + f"Deformable body at '{prim_path}' has no surface indices on its TetMesh prim; " + "cannot sync to visual mesh." + ) + vis_mesh.GetPointsAttr().Set(tet_mesh_prim.GetPointsAttr().Get()) + vis_mesh.GetFaceVertexIndicesAttr().Set(np.asarray(surface_indices).flatten()) + vis_mesh.GetFaceVertexCountsAttr().Set([3] * len(surface_indices)) + else: + sim_mesh = UsdGeom.Mesh(sim_mesh_prim) + vis_mesh.GetFaceVertexIndicesAttr().Set(sim_mesh.GetFaceVertexIndicesAttr().Get()) + vis_mesh.GetFaceVertexCountsAttr().Set(sim_mesh.GetFaceVertexCountsAttr().Get()) + + # bind visual to sim mesh by applying bind pose deformable pose API + purposes = ["bindPose"] + vis_mesh_prim.ApplyAPI("OmniPhysicsDeformablePoseAPI", "default") + vis_mesh_prim.CreateAttribute("deformablePose:default:omniphysics:purposes", Sdf.ValueTypeNames.TokenArray).Set( + purposes + ) + points = UsdGeom.PointBased(vis_mesh_prim).GetPointsAttr().Get() + vis_mesh_prim.CreateAttribute("deformablePose:default:omniphysics:points", Sdf.ValueTypeNames.Point3fArray).Set( + points + ) + + sim_mesh_prim.ApplyAPI("OmniPhysicsDeformablePoseAPI", "default") + sim_mesh_prim.CreateAttribute("deformablePose:default:omniphysics:purposes", Sdf.ValueTypeNames.TokenArray).Set( + purposes + ) + + # disable simulation mesh for rendering + UsdGeom.Imageable(sim_mesh_prim).GetPurposeAttr().Set(UsdGeom.Tokens.guide) + + # apply deformable body api + if not root_prim.ApplyAPI("OmniPhysicsDeformableBodyAPI"): + raise RuntimeError(f"Failed to set deformable body API on prim '{prim_path}'.") + + # set deformable body properties + modify_deformable_body_properties(prim_path, cfg, stage) + + +@apply_nested +def modify_deformable_body_properties( + prim_path: str, cfg: schemas_cfg.DeformableBodyPropertiesBaseCfg, stage: Usd.Stage | None = None +): + """Modify deformable body parameters for a deformable body prim. + + A `deformable body`_ is a single body (either surface or volume deformable) that can be simulated by PhysX + or Newton. Unlike rigid bodies, deformable bodies support relative motion of the nodes in the mesh. + Consequently, they can be used to simulate deformations under applied forces. + + PhysX deformable body simulation employs Finite Element Analysis (FEA) to simulate the deformations of the mesh. + It uses two meshes to represent the deformable body: + + 1. **Simulation mesh**: This mesh is used for the simulation and is the one that is deformed by the solver. + 2. **Collision mesh**: This mesh only needs to match the surface of the simulation mesh and is used for + collision detection. + + For most applications, we assume that the above two meshes are computed from the "render mesh" of the deformable + body. The render mesh is the mesh that is visible in the scene and is used for rendering purposes. It is composed + of triangles, while the simulation mesh is composed of tetrahedrons for volume deformables, + and triangles for surface deformables. + + We apply similar design choices to the simulation in Newton with a separate visual, simulation and collision mesh. + + .. caution:: + The deformable body schema is still under development by the Omniverse team. The current implementation + works with the PhysX schemas shipped with Isaac Sim 6.0.0 onwards. It may change in future releases. + + .. note:: + This function is decorated with :func:`apply_nested` that sets the properties to all the prims + (that have the schema applied on them) under the input prim path. + + .. _deformable body: https://nvidia-omniverse.github.io/PhysX/physx/5.6.1/docs/DeformableVolume.html + .. _PhysxDeformableBodyAPI: https://docs.omniverse.nvidia.com/kit/docs/omni_usd_schema_physics/latest/physxschema/annotated.html + + Args: + prim_path: The prim path to the deformable body. + cfg: The configuration for the deformable body. + stage: The stage where to find the prim. Defaults to None, in which case the + current stage is used. + + Returns: + True if the properties were successfully set, False otherwise. + """ + # get stage handle + if stage is None: + stage = get_current_stage() + + # get deformable-body USD prim + deformable_body_prim = stage.GetPrimAtPath(prim_path) + # check if the prim is valid + if not deformable_body_prim.IsValid(): + return False + # check if deformable body API is applied + if "OmniPhysicsDeformableBodyAPI" not in deformable_body_prim.GetAppliedSchemas(): + return False + + # build cfg dict from dataclass fields only; USD routing is driven by the + # declaring classes' ``_usd_namespace`` / ``_usd_applied_schema`` metadata. + cfg_dict = {f.name: getattr(cfg, f.name) for f in dataclasses.fields(cfg)} + + if cfg_dict.get("kinematic_enabled"): + logger.warning( + "Kinematic deformable bodies are not fully supported in the current version of Omni Physics. " + "Setting kinematic_enabled to True may lead to unexpected behavior." + ) + + _apply_namespaced_schemas(deformable_body_prim, cfg, cfg_dict) + # success + return True diff --git a/source/isaaclab/isaaclab/sim/schemas/schemas_cfg.py b/source/isaaclab/isaaclab/sim/schemas/schemas_cfg.py index b154d91f4cc..d6dc99a8474 100644 --- a/source/isaaclab/isaaclab/sim/schemas/schemas_cfg.py +++ b/source/isaaclab/isaaclab/sim/schemas/schemas_cfg.py @@ -22,8 +22,9 @@ "PhysxJointDrivePropertiesCfg", "CollisionPropertiesCfg", "PhysxCollisionPropertiesCfg", - "PhysXCollisionPropertiesCfg", + "DeformableBodyPropertiesCfg", "PhysxDeformableCollisionPropertiesCfg", + "PhysxDeformableBodyPropertiesCfg", "ArticulationRootPropertiesCfg", "PhysxArticulationRootPropertiesCfg", "MeshCollisionPropertiesCfg", @@ -543,3 +544,14 @@ class BoundingSpherePropertiesCfg(MeshCollisionBaseCfg): mesh_approximation_name: str = "boundingSphere" """Name of mesh collision approximation method. Default: "boundingSphere".""" + + +@configclass +class DeformableBodyPropertiesBaseCfg: + """Base deformable body properties for backend-specific extensions. + + This class is currently empty. It will be populated once the USD deformable + schemas can be unified more cleanly between physics backends. + """ + + pass diff --git a/source/isaaclab/isaaclab/sim/simulation_context.py b/source/isaaclab/isaaclab/sim/simulation_context.py index 175961fcd38..18436947729 100644 --- a/source/isaaclab/isaaclab/sim/simulation_context.py +++ b/source/isaaclab/isaaclab/sim/simulation_context.py @@ -11,6 +11,7 @@ import traceback from collections.abc import Iterator from contextlib import contextmanager +from dataclasses import fields from typing import TYPE_CHECKING, Any import toml @@ -29,7 +30,7 @@ resolve_scene_data_requirements, ) from isaaclab.renderers.render_context import RenderContext -from isaaclab.scene.scene_data_provider import SceneDataProvider +from isaaclab.scene_data import SceneDataProvider from isaaclab.sim.utils import create_new_stage from isaaclab.utils.string import clear_resolve_matching_names_cache from isaaclab.utils.version import has_kit @@ -441,7 +442,9 @@ def _create_default_visualizer_configs(self, requested_visualizers: list[str]) - continue mod = importlib.import_module(f"isaaclab_visualizers.{viz_type}") cfg_cls = getattr(mod, cfg_class_names[viz_type]) - default_configs.append(cfg_cls()) + cfg = cfg_cls() + self._apply_default_visualizer_cfg(cfg) + default_configs.append(cfg) except (ImportError, ModuleNotFoundError) as exc: # isaaclab_visualizers is optional; log once at warning level if "isaaclab_visualizers" in str(exc): @@ -461,6 +464,16 @@ def _create_default_visualizer_configs(self, requested_visualizers: list[str]) - logger.error(f"[SimulationContext] Failed to create default config for visualizer '{viz_type}': {exc}") return default_configs + def _apply_default_visualizer_cfg(self, cfg: Any) -> None: + """Apply shared default visualizer settings to a backend-specific config.""" + default_cfg = getattr(self.cfg, "default_visualizer_cfg", None) + if default_cfg is None: + return + for field in fields(default_cfg): + if field.name == "visualizer_type" or not hasattr(cfg, field.name): + continue + setattr(cfg, field.name, getattr(default_cfg, field.name)) + def _get_cli_visualizer_types(self) -> list[str]: """Return list of visualizer types requested via CLI (setting).""" requested = self.get_setting("/isaaclab/visualizer/types") @@ -632,9 +645,21 @@ def initialize_visualizers(self) -> None: viz.set_camera_view(eye, target) self._pending_camera_view = None + if not self._visualizers and self._scene_data_provider is not None: + close_provider = getattr(self._scene_data_provider, "close", None) + if callable(close_provider): + close_provider() + self._scene_data_provider = None + def get_scene_data_provider(self) -> SceneDataProvider: return self._scene_data_provider + def register_interactive_scene(self, scene) -> None: + """Register the active scene so scene data providers can expose scene-owned sensors.""" + self._interactive_scene = scene + if self._scene_data_provider is not None: + self._scene_data_provider.set_interactive_scene(scene) + def get_scene_data_requirements(self) -> SceneDataRequirement: """Return scene-data requirements resolved from visualizers/renderers.""" return self._scene_data_requirements @@ -761,6 +786,9 @@ def update_visualizers(self, dt: float, skip_app_pumping: bool = False) -> None: if not self._visualizers: return + for viz in self._visualizers: + viz.flush_startup_messages() + if self._should_forward_before_visualizer_update(): self.physics_manager.forward() @@ -769,11 +797,6 @@ def update_visualizers(self, dt: float, skip_app_pumping: bool = False) -> None: if any(viz.supports_markers() for viz in self._visualizers): self.vis_marker_registry.dispatch_callbacks() - # Marker callbacks update VisualizationMarkers state; visualizer step() - # consumes that state later in this method. - if any(viz.supports_markers() for viz in self._visualizers): - self.vis_marker_registry.dispatch_callbacks() - visualizers_to_remove = [] for viz in self._visualizers: try: diff --git a/source/isaaclab/isaaclab/sim/spawners/__init__.pyi b/source/isaaclab/isaaclab/sim/spawners/__init__.pyi index dae1a432b47..c936d166ba3 100644 --- a/source/isaaclab/isaaclab/sim/spawners/__init__.pyi +++ b/source/isaaclab/isaaclab/sim/spawners/__init__.pyi @@ -6,6 +6,7 @@ __all__ = [ "SpawnerCfg", "RigidObjectSpawnerCfg", + "DeformableObjectSpawnerCfg", "spawn_from_mjcf", "spawn_from_urdf", "spawn_from_usd", @@ -24,8 +25,13 @@ __all__ = [ "LightCfg", "SphereLightCfg", "spawn_rigid_body_material", + "spawn_deformable_body_material", "PhysicsMaterialCfg", "RigidBodyMaterialCfg", + "DeformableBodyMaterialBaseCfg", + "DeformableBodyMaterialCfg", + "SurfaceDeformableBodyMaterialBaseCfg", + "SurfaceDeformableBodyMaterialCfg", "spawn_from_mdl_file", "spawn_preview_surface", "GlassMdlCfg", @@ -36,15 +42,15 @@ __all__ = [ "spawn_mesh_cone", "spawn_mesh_cuboid", "spawn_mesh_cylinder", + "spawn_mesh_rectangle", "spawn_mesh_sphere", - "spawn_mesh_square", "MeshCapsuleCfg", "MeshCfg", "MeshConeCfg", "MeshCuboidCfg", "MeshCylinderCfg", + "MeshRectangleCfg", "MeshSphereCfg", - "MeshSquareCfg", "spawn_camera", "spawn_sensor_frame", "FisheyeCameraCfg", @@ -67,7 +73,7 @@ __all__ = [ "MultiUsdFileCfg", ] -from .spawner_cfg import SpawnerCfg, RigidObjectSpawnerCfg +from .spawner_cfg import SpawnerCfg, RigidObjectSpawnerCfg, DeformableObjectSpawnerCfg from .from_files import ( spawn_from_mjcf, spawn_from_urdf, @@ -91,8 +97,13 @@ from .lights import ( ) from .materials import ( spawn_rigid_body_material, + spawn_deformable_body_material, PhysicsMaterialCfg, RigidBodyMaterialCfg, + DeformableBodyMaterialBaseCfg, + DeformableBodyMaterialCfg, + SurfaceDeformableBodyMaterialBaseCfg, + SurfaceDeformableBodyMaterialCfg, spawn_from_mdl_file, spawn_preview_surface, GlassMdlCfg, @@ -105,14 +116,14 @@ from .meshes import ( spawn_mesh_cone, spawn_mesh_cuboid, spawn_mesh_cylinder, + spawn_mesh_rectangle, spawn_mesh_sphere, - spawn_mesh_square, MeshCapsuleCfg, MeshCfg, MeshConeCfg, MeshCuboidCfg, MeshCylinderCfg, - MeshSquareCfg, + MeshRectangleCfg, MeshSphereCfg, ) from .sensors import spawn_camera, spawn_sensor_frame, FisheyeCameraCfg, PinholeCameraCfg, SensorFrameCfg diff --git a/source/isaaclab/isaaclab/sim/spawners/from_files/from_files.py b/source/isaaclab/isaaclab/sim/spawners/from_files/from_files.py index 18ffcaf37c9..9367ef3cc80 100644 --- a/source/isaaclab/isaaclab/sim/spawners/from_files/from_files.py +++ b/source/isaaclab/isaaclab/sim/spawners/from_files/from_files.py @@ -13,14 +13,10 @@ from filelock import FileLock -# deformables only supported on PhysX backend -from isaaclab_physx.sim import schemas as schemas_physx -from isaaclab_physx.sim.spawners.materials import SurfaceDeformableBodyMaterialCfg - from pxr import Gf, Sdf, Usd, UsdGeom from isaaclab.sim import converters, schemas -from isaaclab.sim.spawners.materials import RigidBodyMaterialCfg +from isaaclab.sim.spawners.materials import RigidBodyMaterialCfg, SurfaceDeformableBodyMaterialBaseCfg from isaaclab.sim.utils import ( add_labels, bind_physics_material, @@ -387,15 +383,17 @@ def _spawn_from_usd_file( # define deformable body properties, or modify if deformable body API is present (PhysX only) if cfg.deformable_props is not None: prim = stage.GetPrimAtPath(prim_path) - deformable_type = "surface" if isinstance(cfg.physics_material, SurfaceDeformableBodyMaterialCfg) else "volume" + deformable_type = ( + "surface" if isinstance(cfg.physics_material, SurfaceDeformableBodyMaterialBaseCfg) else "volume" + ) if "OmniPhysicsDeformableBodyAPI" in prim.GetAppliedSchemas(): - schemas_physx.modify_deformable_body_properties(prim_path, cfg.deformable_props, stage) + schemas.modify_deformable_body_properties(prim_path, cfg.deformable_props, stage) else: - schemas_physx.define_deformable_body_properties(prim_path, cfg.deformable_props, stage, deformable_type) + schemas.define_deformable_body_properties(prim_path, cfg.deformable_props, stage, deformable_type) if cfg.mass_props is not None: raise ValueError( """MassPropertiesCfg are not supported for deformable bodies - and should be set through DeformableBodyPropertiesCfg(mass=).""" + and should be set through deformable_props with mass=.""" ) # apply visual material diff --git a/source/isaaclab/isaaclab/sim/spawners/from_files/from_files_cfg.py b/source/isaaclab/isaaclab/sim/spawners/from_files/from_files_cfg.py index 1d61cde5725..22857f8d45b 100644 --- a/source/isaaclab/isaaclab/sim/spawners/from_files/from_files_cfg.py +++ b/source/isaaclab/isaaclab/sim/spawners/from_files/from_files_cfg.py @@ -8,12 +8,9 @@ from collections.abc import Callable from dataclasses import MISSING -# deformables only supported on PhysX backend -from isaaclab_physx.sim.spawners.spawner_cfg import DeformableObjectSpawnerCfg - from isaaclab.sim import converters, schemas from isaaclab.sim.spawners import materials -from isaaclab.sim.spawners.spawner_cfg import RigidObjectSpawnerCfg, SpawnerCfg +from isaaclab.sim.spawners.spawner_cfg import DeformableObjectSpawnerCfg, RigidObjectSpawnerCfg, SpawnerCfg from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR from isaaclab.utils.configclass import configclass diff --git a/source/isaaclab/isaaclab/sim/spawners/materials/__init__.py b/source/isaaclab/isaaclab/sim/spawners/materials/__init__.py index 3158625219c..0df8e552ea2 100644 --- a/source/isaaclab/isaaclab/sim/spawners/materials/__init__.py +++ b/source/isaaclab/isaaclab/sim/spawners/materials/__init__.py @@ -60,8 +60,12 @@ # Resolved lazily on first access so importing ``isaaclab.sim.spawners.materials`` does # not require ``isaaclab_physx`` to be installed. _PHYSX_FORWARDS = frozenset({ + "DeformableBodyMaterialCfg", "RigidBodyMaterialCfg", + "SurfaceDeformableBodyMaterialCfg", "PhysxRigidBodyMaterialCfg", + "PhysxDeformableBodyMaterialCfg", + "PhysxSurfaceDeformableBodyMaterialCfg", }) diff --git a/source/isaaclab/isaaclab/sim/spawners/materials/__init__.pyi b/source/isaaclab/isaaclab/sim/spawners/materials/__init__.pyi index 0dd023c2998..7ce7815955c 100644 --- a/source/isaaclab/isaaclab/sim/spawners/materials/__init__.pyi +++ b/source/isaaclab/isaaclab/sim/spawners/materials/__init__.pyi @@ -5,8 +5,13 @@ __all__ = [ "spawn_rigid_body_material", + "spawn_deformable_body_material", "PhysicsMaterialCfg", "RigidBodyMaterialBaseCfg", + "DeformableBodyMaterialBaseCfg", + "DeformableBodyMaterialCfg", + "SurfaceDeformableBodyMaterialBaseCfg", + "SurfaceDeformableBodyMaterialCfg", "spawn_from_mdl_file", "spawn_preview_surface", "GlassMdlCfg", @@ -15,10 +20,14 @@ __all__ = [ "VisualMaterialCfg", ] -from .physics_materials import spawn_rigid_body_material +from .physics_materials import spawn_rigid_body_material, spawn_deformable_body_material from .physics_materials_cfg import ( PhysicsMaterialCfg, RigidBodyMaterialBaseCfg, + DeformableBodyMaterialBaseCfg, + DeformableBodyMaterialCfg, + SurfaceDeformableBodyMaterialBaseCfg, + SurfaceDeformableBodyMaterialCfg, ) from .visual_materials import spawn_from_mdl_file, spawn_preview_surface from .visual_materials_cfg import GlassMdlCfg, MdlFileCfg, PreviewSurfaceCfg, VisualMaterialCfg diff --git a/source/isaaclab/isaaclab/sim/spawners/materials/physics_materials.py b/source/isaaclab/isaaclab/sim/spawners/materials/physics_materials.py index b76077b38be..8c12bee9442 100644 --- a/source/isaaclab/isaaclab/sim/spawners/materials/physics_materials.py +++ b/source/isaaclab/isaaclab/sim/spawners/materials/physics_materials.py @@ -6,7 +6,6 @@ from __future__ import annotations import dataclasses -from typing import TYPE_CHECKING from pxr import Usd, UsdPhysics, UsdShade @@ -14,8 +13,7 @@ from isaaclab.sim.utils import clone from isaaclab.sim.utils.stage import get_current_stage -if TYPE_CHECKING: - from . import physics_materials_cfg +from . import physics_materials_cfg @clone @@ -76,3 +74,49 @@ def spawn_rigid_body_material(prim_path: str, cfg: physics_materials_cfg.RigidBo # return the prim return prim + + +@clone +def spawn_deformable_body_material( + prim_path: str, cfg: physics_materials_cfg.DeformableBodyMaterialBaseCfg +) -> Usd.Prim: + """Create material with deformable-body physics properties. + + Deformable body materials are used to define the physical properties to meshes of a deformable body. These + include the friction and deformable body properties. For more information on deformable body material, + please refer to the documentation on `PxFEMSoftBodyMaterial`_. + + .. note:: + This function is decorated with :func:`clone` that resolves prim path into list of paths + if the input prim path is a regex pattern. This is done to support spawning multiple assets + from a single and cloning the USD prim at the given path expression. + + Args: + prim_path: The prim path or pattern to spawn the asset at. If the prim path is a regex pattern, + then the asset is spawned at all the matching prim paths. + cfg: The configuration for the physics material. + + Returns: + The spawned deformable body material prim. + + Raises: + ValueError: When a prim already exists at the specified prim path and is not a material. + + .. _PxFEMSoftBodyMaterial: https://nvidia-omniverse.github.io/PhysX/physx/5.4.1/_api_build/structPxFEMSoftBodyMaterialModel.html + """ + # get stage handle + stage = get_current_stage() + + # create material prim if no prim exists + if not stage.GetPrimAtPath(prim_path).IsValid(): + _ = UsdShade.Material.Define(stage, prim_path) + + # obtain prim + prim = stage.GetPrimAtPath(prim_path) + # check if prim is a material + if not prim.IsA(UsdShade.Material): + raise ValueError(f"A prim already exists at path: '{prim_path}' but is not a material.") + cfg_dict = {f.name: getattr(cfg, f.name) for f in dataclasses.fields(cfg) if f.name != "func"} + _apply_namespaced_schemas(prim, cfg, cfg_dict) + # return the prim + return prim diff --git a/source/isaaclab/isaaclab/sim/spawners/materials/physics_materials_cfg.py b/source/isaaclab/isaaclab/sim/spawners/materials/physics_materials_cfg.py index 886b049b77b..5c88731cf8d 100644 --- a/source/isaaclab/isaaclab/sim/spawners/materials/physics_materials_cfg.py +++ b/source/isaaclab/isaaclab/sim/spawners/materials/physics_materials_cfg.py @@ -15,7 +15,16 @@ # Resolved lazily so callers using ``from isaaclab.sim.spawners.materials.physics_materials_cfg # import RigidBodyMaterialCfg`` continue to work without importing ``isaaclab_physx`` at module # load time. -_PHYSX_FORWARDS = frozenset({"RigidBodyMaterialCfg", "PhysxRigidBodyMaterialCfg"}) +_PHYSX_FORWARDS = frozenset( + { + "DeformableBodyMaterialCfg", + "RigidBodyMaterialCfg", + "SurfaceDeformableBodyMaterialCfg", + "PhysxRigidBodyMaterialCfg", + "PhysxDeformableBodyMaterialCfg", + "PhysxSurfaceDeformableBodyMaterialCfg", + } +) def __getattr__(name): @@ -78,3 +87,19 @@ class RigidBodyMaterialBaseCfg(PhysicsMaterialCfg): restitution: float = 0.0 """The restitution coefficient. Defaults to 0.0.""" + + +@configclass +class DeformableBodyMaterialBaseCfg(PhysicsMaterialCfg): + """Base physics material parameters for volume deformable bodies. + + Backend-specific subclasses provide the material fields and spawning function + through :attr:`func`. + """ + + func: Callable | str | None = None + + +@configclass +class SurfaceDeformableBodyMaterialBaseCfg(DeformableBodyMaterialBaseCfg): + """Base physics material parameters for surface deformable bodies.""" diff --git a/source/isaaclab/isaaclab/sim/spawners/meshes/__init__.pyi b/source/isaaclab/isaaclab/sim/spawners/meshes/__init__.pyi index 06490befb33..c853dca4d1d 100644 --- a/source/isaaclab/isaaclab/sim/spawners/meshes/__init__.pyi +++ b/source/isaaclab/isaaclab/sim/spawners/meshes/__init__.pyi @@ -8,15 +8,15 @@ __all__ = [ "spawn_mesh_cone", "spawn_mesh_cuboid", "spawn_mesh_cylinder", + "spawn_mesh_rectangle", "spawn_mesh_sphere", - "spawn_mesh_square", "MeshCapsuleCfg", "MeshCfg", "MeshConeCfg", "MeshCuboidCfg", "MeshCylinderCfg", + "MeshRectangleCfg", "MeshSphereCfg", - "MeshSquareCfg", ] from .meshes import ( @@ -24,8 +24,8 @@ from .meshes import ( spawn_mesh_cone, spawn_mesh_cuboid, spawn_mesh_cylinder, + spawn_mesh_rectangle, spawn_mesh_sphere, - spawn_mesh_square, ) from .meshes_cfg import ( MeshCapsuleCfg, @@ -33,6 +33,6 @@ from .meshes_cfg import ( MeshConeCfg, MeshCuboidCfg, MeshCylinderCfg, + MeshRectangleCfg, MeshSphereCfg, - MeshSquareCfg, ) diff --git a/source/isaaclab/isaaclab/sim/spawners/meshes/meshes.py b/source/isaaclab/isaaclab/sim/spawners/meshes/meshes.py index 75ead1c8fb6..cfc7f51b9ff 100644 --- a/source/isaaclab/isaaclab/sim/spawners/meshes/meshes.py +++ b/source/isaaclab/isaaclab/sim/spawners/meshes/meshes.py @@ -11,16 +11,12 @@ import trimesh import trimesh.transformations -# deformables only supported on PhysX backend -from isaaclab_physx.sim import schemas as schemas_physx -from isaaclab_physx.sim.spawners.materials import DeformableBodyMaterialCfg, SurfaceDeformableBodyMaterialCfg - from pxr import Usd, UsdPhysics from isaaclab.sim import schemas from isaaclab.sim.utils import bind_physics_material, bind_visual_material, clone, create_prim, get_current_stage -from ..materials import RigidBodyMaterialCfg +from ..materials import DeformableBodyMaterialBaseCfg, RigidBodyMaterialCfg, SurfaceDeformableBodyMaterialBaseCfg if TYPE_CHECKING: from . import meshes_cfg @@ -261,14 +257,14 @@ def spawn_mesh_cone( @clone -def spawn_mesh_square( +def spawn_mesh_rectangle( prim_path: str, - cfg: meshes_cfg.MeshSquareCfg, + cfg: meshes_cfg.MeshRectangleCfg, translation: tuple[float, float, float] | None = None, orientation: tuple[float, float, float, float] | None = None, **kwargs, ) -> Usd.Prim: - """Create a USD-Mesh 2D square prim with the given attributes. + """Create a USD-Mesh 2D rectangle prim with the given attributes. .. note:: This function is decorated with :func:`clone` that resolves prim path into list of paths @@ -294,12 +290,13 @@ def spawn_mesh_square( # create a 2D triangle mesh grid from omni.physx.scripts import deformableUtils - vertices, faces = deformableUtils.create_triangle_mesh_square(cfg.resolution[0], cfg.resolution[1], scale=cfg.size) + vertices, faces = deformableUtils.create_triangle_mesh_square(cfg.resolution[0], cfg.resolution[1], scale=1.0) + vertices = np.array([(v[0] * cfg.size[0], v[1] * cfg.size[1], v[2]) for v in vertices], dtype=np.float32) grid = trimesh.Trimesh(vertices=vertices, faces=np.array(faces).reshape(-1, 3), process=False) # obtain stage handle stage = get_current_stage() - # spawn the square as a mesh + # spawn the rectangle as a mesh _spawn_mesh_geom_from_mesh(prim_path, cfg, grid, translation, orientation, None, stage=stage) # return the prim return stage.GetPrimAtPath(prim_path) @@ -367,7 +364,7 @@ def _spawn_mesh_geom_from_mesh( raise ValueError("Cannot use both deformable and collision properties at the same time.") # check material types are correct if cfg.deformable_props is not None and cfg.physics_material is not None: - if not isinstance(cfg.physics_material, DeformableBodyMaterialCfg): + if not isinstance(cfg.physics_material, DeformableBodyMaterialBaseCfg): raise ValueError("Deformable properties require a deformable physics material.") if cfg.rigid_props is not None and cfg.physics_material is not None: if not isinstance(cfg.physics_material, RigidBodyMaterialCfg): @@ -393,14 +390,16 @@ def _spawn_mesh_geom_from_mesh( if cfg.deformable_props is not None: # apply deformable body properties - deformable_type = "surface" if isinstance(cfg.physics_material, SurfaceDeformableBodyMaterialCfg) else "volume" - schemas_physx.define_deformable_body_properties( + deformable_type = ( + "surface" if isinstance(cfg.physics_material, SurfaceDeformableBodyMaterialBaseCfg) else "volume" + ) + schemas.define_deformable_body_properties( prim_path, cfg.deformable_props, stage=stage, deformable_type=deformable_type ) if cfg.mass_props is not None: raise ValueError( """MassPropertiesCfg are not supported for deformable bodies - and should be set through DeformableBodyPropertiesCfg(mass=).""" + and should be set through deformable_props with mass=.""" ) elif cfg.collision_props is not None: # decide on type of collision approximation based on the mesh diff --git a/source/isaaclab/isaaclab/sim/spawners/meshes/meshes_cfg.py b/source/isaaclab/isaaclab/sim/spawners/meshes/meshes_cfg.py index 6bdf9ebf9ba..c6eb26507d3 100644 --- a/source/isaaclab/isaaclab/sim/spawners/meshes/meshes_cfg.py +++ b/source/isaaclab/isaaclab/sim/spawners/meshes/meshes_cfg.py @@ -9,11 +9,8 @@ from dataclasses import MISSING from typing import Literal -# deformables only supported on PhysX backend -from isaaclab_physx.sim.spawners.spawner_cfg import DeformableObjectSpawnerCfg - from isaaclab.sim.spawners import materials -from isaaclab.sim.spawners.spawner_cfg import RigidObjectSpawnerCfg +from isaaclab.sim.spawners.spawner_cfg import DeformableObjectSpawnerCfg, RigidObjectSpawnerCfg from isaaclab.utils.configclass import configclass @@ -146,15 +143,15 @@ class MeshConeCfg(MeshCfg): @configclass -class MeshSquareCfg(MeshCfg): - """Configuration parameters for a 2D square mesh prim. +class MeshRectangleCfg(MeshCfg): + """Configuration parameters for a 2D rectangle mesh prim. - See :meth:`spawn_mesh_square` for more information. + See :meth:`spawn_mesh_rectangle` for more information. """ - func: Callable | str = "{DIR}.meshes:spawn_mesh_square" + func: Callable | str = "{DIR}.meshes:spawn_mesh_rectangle" - size: float = MISSING - """Edge length of the square (in m).""" + size: tuple[float, float] = MISSING + """Edge lengths of the rectangle along the X and Y axes [m].""" resolution: tuple[int, int] = (5, 5) - """Resolution of the square (in elements/edges per side).""" + """Resolution of the rectangle (in elements/edges per side).""" diff --git a/source/isaaclab/isaaclab/sim/spawners/spawner_cfg.py b/source/isaaclab/isaaclab/sim/spawners/spawner_cfg.py index 3cc632e2c0d..3f1eef72a2f 100644 --- a/source/isaaclab/isaaclab/sim/spawners/spawner_cfg.py +++ b/source/isaaclab/isaaclab/sim/spawners/spawner_cfg.py @@ -100,3 +100,25 @@ class RigidObjectSpawnerCfg(SpawnerCfg): This adds the PhysxContactReporter API to all the rigid bodies in the given prim path and its children. """ + + +@configclass +class DeformableObjectSpawnerCfg(SpawnerCfg): + """Configuration parameters for spawning a deformable asset. + + Unlike rigid objects, deformable objects are affected by forces and can deform when subjected to + external forces. This class is used to configure the properties of the deformable object. + + Deformable bodies don't have a separate collision mesh. The collision mesh is the same as the visual mesh. + The collision properties such as rest and collision offsets are specified in the :attr:`deformable_props`. + + Note: + By default, all properties are set to None. This means that no properties will be added or modified + to the prim outside of the properties available by default when spawning the prim. + """ + + mass_props: schemas.MassPropertiesCfg | None = None + """Mass properties.""" + + deformable_props: schemas.DeformableBodyPropertiesBaseCfg | None = None + """Deformable body properties.""" diff --git a/source/isaaclab/isaaclab/sim/spawners/wrappers/wrappers_cfg.py b/source/isaaclab/isaaclab/sim/spawners/wrappers/wrappers_cfg.py index dafe5beb812..285c1765016 100644 --- a/source/isaaclab/isaaclab/sim/spawners/wrappers/wrappers_cfg.py +++ b/source/isaaclab/isaaclab/sim/spawners/wrappers/wrappers_cfg.py @@ -3,27 +3,10 @@ # # SPDX-License-Identifier: BSD-3-Clause -import warnings from dataclasses import MISSING -# deformables only supported in PhysX backend -try: - from isaaclab_physx.sim.spawners.spawner_cfg import DeformableObjectSpawnerCfg -except ImportError as e: - warnings.warn( - f"""Could not import DeformableObjectSpawnerCfg, is isaaclab_physx installed? - Safe to ignore if using newton only. Complete exception: {e}""" - ) - # import dummy class to avoid errors in type hints - from isaaclab.utils.configclass import configclass - - @configclass - class DeformableObjectSpawnerCfg: - deformable_props = None - - from isaaclab.sim.spawners.from_files import UsdFileCfg -from isaaclab.sim.spawners.spawner_cfg import RigidObjectSpawnerCfg, SpawnerCfg +from isaaclab.sim.spawners.spawner_cfg import DeformableObjectSpawnerCfg, RigidObjectSpawnerCfg, SpawnerCfg from isaaclab.utils.configclass import configclass diff --git a/source/isaaclab/isaaclab/visualizers/base_visualizer.py b/source/isaaclab/isaaclab/visualizers/base_visualizer.py index 92f8c37ce94..b10d9b509f2 100644 --- a/source/isaaclab/isaaclab/visualizers/base_visualizer.py +++ b/source/isaaclab/isaaclab/visualizers/base_visualizer.py @@ -8,19 +8,23 @@ from __future__ import annotations import logging +import math import random import re from abc import ABC, abstractmethod from typing import TYPE_CHECKING, Any +from urllib.parse import urlparse if TYPE_CHECKING: - from isaaclab.scene.scene_data_provider import SceneDataProvider + from isaaclab.scene_data import SceneDataProvider from .visualizer_cfg import VisualizerCfg logger = logging.getLogger(__name__) +_USD_DEFAULT_VERTICAL_APERTURE_MM = 15.2908 + class BaseVisualizer(ABC): """Base class for all visualizer backends. @@ -38,6 +42,7 @@ def __init__(self, cfg: VisualizerCfg): self._scene_data_provider = None self._is_initialized = False self._is_closed = False + self._deferred_startup_messages: list[str] = [] @abstractmethod def initialize(self, scene_data_provider: SceneDataProvider) -> None: @@ -48,6 +53,13 @@ def initialize(self, scene_data_provider: SceneDataProvider) -> None: """ raise NotImplementedError + def _set_scene_data_provider(self, scene_data_provider: SceneDataProvider) -> SceneDataProvider: + """Store the scene data provider shared by all visualizer backends.""" + if scene_data_provider is None: + raise RuntimeError(f"{self.__class__.__name__} requires a scene_data_provider.") + self._scene_data_provider = scene_data_provider + return scene_data_provider + @abstractmethod def step(self, dt: float) -> None: """Update visualization for one step. @@ -188,6 +200,13 @@ def _resolve_cfg_camera_pose( lookat = tuple(float(v) for v in self.cfg.lookat) return eye, lookat + def _focal_length_to_vertical_fov_degrees(self) -> float: + """Convert cfg focal length to vertical FOV using USD's default aperture.""" + focal_length = float(self.cfg.focal_length) + if focal_length <= 0.0: + raise ValueError("VisualizerCfg.focal_length must be positive.") + return math.degrees(2.0 * math.atan(_USD_DEFAULT_VERTICAL_APERTURE_MM / (2.0 * focal_length))) + def _resolve_camera_pose_from_usd_path( self, usd_path: str ) -> tuple[tuple[float, float, float], tuple[float, float, float]] | None: @@ -308,7 +327,43 @@ def _log_initialization_table(self, logger: logging.Logger, title: str, rows: li table.align["Value"] = "l" for key, value in rows: table.add_row([key, value]) - logger.info("Visualizer initialization:\n%s", table.get_string()) + logger.debug("Visualizer initialization:\n%s", table.get_string()) + + def _log_viewer_url( + self, + visualizer_name: str, + viewer_url: str, + ) -> None: + """Queue a visible browser URL block for web-based visualizers. + + Args: + visualizer_name: Name of the visualizer exposing the URL. + viewer_url: Browser URL for the visualizer. + """ + parsed_url = urlparse(viewer_url) + visualizer_label = visualizer_name.removesuffix("Visualizer").lower() + title = f" {visualizer_label} (listening *:{parsed_url.port}) " if parsed_url.port else f" {visualizer_label} " + label = "URL" + label_width = len(label) + value_width = max(len(viewer_url), len(title) + 2, 21) + inner_width = label_width + value_width + 9 + left_rule_width = max((inner_width - len(title)) // 2, 1) + right_rule_width = max(inner_width - len(title) - left_rule_width, 1) + + lines = [ + f"╭{'─' * left_rule_width}{title}{'─' * right_rule_width}╮", + f"│{' ' * (label_width + 4)}╷{' ' * (value_width + 4)}│", + f"│ {label:<{label_width}} │ {viewer_url:<{value_width}} │", + f"│{' ' * (label_width + 4)}╵{' ' * (value_width + 4)}│", + f"╰{'─' * inner_width}╯", + ] + self._deferred_startup_messages.append("\n" + "\n".join(lines) + "\n") + + def flush_startup_messages(self) -> None: + """Print deferred startup messages immediately before the workflow update loop starts.""" + for message in self._deferred_startup_messages: + print(message, flush=True) + self._deferred_startup_messages.clear() def play(self) -> None: """Handle simulation play/start. No-op by default.""" diff --git a/source/isaaclab/isaaclab/visualizers/visualizer_cfg.py b/source/isaaclab/isaaclab/visualizers/visualizer_cfg.py index 8a8c7ba7a21..e6d84237e8e 100644 --- a/source/isaaclab/isaaclab/visualizers/visualizer_cfg.py +++ b/source/isaaclab/isaaclab/visualizers/visualizer_cfg.py @@ -7,7 +7,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Literal +from typing import TYPE_CHECKING from isaaclab.utils.configclass import configclass @@ -34,17 +34,39 @@ class VisualizerCfg: enable_live_plots: bool = True """Enable live plotting of data.""" - eye: tuple[float, float, float] = (7.5, 7.5, 7.5) - """Initial camera eye position (x, y, z) in world coordinates.""" + eye: tuple[float, float, float] = (4.0, -4.0, 3.0) + """Interactive visualizer camera eye position in world coordinates.""" lookat: tuple[float, float, float] = (0.0, 0.0, 0.0) - """Initial camera look-at point (x, y, z) in world coordinates.""" + """Interactive visualizer camera look-at target in world coordinates.""" - cam_source: Literal["cfg", "prim_path"] = "cfg" - """Camera source mode: 'cfg' uses eye/lookat, 'prim_path' follows a camera prim.""" + focal_length: float = 12.0 + """Camera focal length in millimeters for visualizer camera views.""" - cam_prim_path: str = "/World/envs/env_0/Camera" - """Absolute USD path to a camera prim when cam_source='prim_path'.""" + tiled_cam_view: bool = False + """Enable a non-interactive tiled camera image view.""" + + tiled_cam_num: int = 16 + """Number of camera tiles to show when tiled_cam_env_indices is None, capped at 100.""" + + tiled_cam_env_indices: list[int] | None = None + """Env ids to show in tiled camera view; capped at 100 entries. + + If ``None``, envs are randomly sampled from all visible envs. + """ + + tiled_cam_prim_path: str | None = None + """Existing Isaac Lab Camera sensor prim path to display. + + If ``None``, the visualizer creates generated tiled cameras. If set, it should + point to an existing camera sensor, for example ``"/World/envs/*/Camera"``. + """ + + tiled_cam_eye: tuple[float, float, float] = (4.0, -4.0, 3.0) + """Eye offset from tiled_cam_target_prim_path for generated tiled cameras.""" + + tiled_cam_target_prim_path: str = "/World/envs/*/Robot/base" + """Prim path that generated tiled cameras follow and look at.""" max_visible_envs: int | None = None """Upper bound on how many envs are shown. diff --git a/source/isaaclab/setup.py b/source/isaaclab/setup.py index bc64609de3f..57f732d4c58 100644 --- a/source/isaaclab/setup.py +++ b/source/isaaclab/setup.py @@ -30,6 +30,8 @@ # procedural-generation "trimesh", "pyglet>=2.1.6,<3", + # tetrahedralization for deformable bodies (pinned: >=0.3 unconditionally imports pyvista at package import time) + "pytetwild==0.2.3", # image processing "transformers==4.57.6", "einops", # needed for transformers, doesn't always auto-install diff --git a/source/isaaclab/test/markers/test_visualization_markers.py b/source/isaaclab/test/markers/test_visualization_markers.py index 906c14ccb7d..b9ae8387cf0 100644 --- a/source/isaaclab/test/markers/test_visualization_markers.py +++ b/source/isaaclab/test/markers/test_visualization_markers.py @@ -12,15 +12,24 @@ """Rest everything follows.""" +import isaaclab_visualizers.newton.newton_visualization_markers as newton_markers +import isaaclab_visualizers.newton.newton_visualizer as newton_visualizer +import isaaclab_visualizers.rerun.rerun_visualizer as rerun_visualizer +import isaaclab_visualizers.viser.viser_visualizer as viser_visualizer +import numpy as np import pytest import torch +from isaaclab_visualizers.kit.kit_visualizer import KitVisualizer +from isaaclab_visualizers.kit.kit_visualizer_cfg import KitVisualizerCfg +from isaaclab_visualizers.newton.newton_visualizer_cfg import NewtonVisualizerCfg +from isaaclab_visualizers.rerun.rerun_visualizer_cfg import RerunVisualizerCfg +from isaaclab_visualizers.viser.viser_visualizer_cfg import ViserVisualizerCfg import isaaclab.sim as sim_utils from isaaclab.markers import VisualizationMarkers, VisualizationMarkersCfg from isaaclab.markers.config import FRAME_MARKER_CFG, POSITION_GOAL_MARKER_CFG from isaaclab.sim import SimulationCfg, SimulationContext from isaaclab.utils.math import random_orientation -from isaaclab.utils.timer import Timer @pytest.fixture @@ -40,6 +49,24 @@ def sim(): sim_utils.close_stage() +class _FakeMarkerVisualizer: + def __init__(self, *, enable_markers: bool = True, pumps_app_update: bool = False): + self.cfg = type("Cfg", (), {"enable_markers": enable_markers})() + self._pumps_app_update = pumps_app_update + + def supports_markers(self): + return True + + def pumps_app_update(self): + return self._pumps_app_update + + def stop(self): + pass + + def close(self): + pass + + def test_instantiation(sim): """Test that the class can be initialized properly.""" config = VisualizationMarkersCfg( @@ -54,6 +81,69 @@ def test_instantiation(sim): assert test_marker.num_prototypes == 1 +@pytest.mark.parametrize( + ("is_rendering", "visualizers", "expected_backends"), + [ + (True, [], ["kit"]), + (False, [], []), + (False, [KitVisualizer(KitVisualizerCfg())], ["kit"]), + (False, [newton_visualizer.NewtonVisualizer(NewtonVisualizerCfg())], ["newton"]), + (False, [rerun_visualizer.RerunVisualizer(RerunVisualizerCfg())], ["newton"]), + (False, [viser_visualizer.ViserVisualizer(ViserVisualizerCfg())], ["newton"]), + ], +) +def test_marker_backend_selection(monkeypatch, is_rendering: bool, visualizers: list, expected_backends: list[str]): + """Marker backend selection follows rendering state and active visualizer type.""" + marker = object.__new__(VisualizationMarkers) + marker._backends = [] + fake_sim = type("FakeSim", (), {"is_rendering": is_rendering, "visualizers": visualizers})() + + monkeypatch.setattr(sim_utils.SimulationContext, "instance", staticmethod(lambda: fake_sim)) + monkeypatch.setattr(VisualizationMarkers, "_ensure_kit_backend", lambda self: self._backends.append("kit")) + monkeypatch.setattr(VisualizationMarkers, "_ensure_newton_backend", lambda self: self._backends.append("newton")) + + marker._ensure_backends_initialized() + + assert marker._backends == expected_backends + + +def test_rendering_context_authors_visible_usd_point_instancer(sim): + """Rendering-active contexts should create visible USD marker prims.""" + from pxr import UsdGeom + + sim._has_offscreen_render = True + config = VisualizationMarkersCfg( + prim_path="/World/Visuals/rendered_marker", + markers={ + "failure": sim_utils.CuboidCfg( + size=(0.1, 0.1, 0.1), + visual_material=sim_utils.PreviewSurfaceCfg(diffuse_color=(0.25, 0.15, 0.15)), + visible=True, + ), + "success": sim_utils.CuboidCfg( + size=(0.1, 0.1, 0.1), + visual_material=sim_utils.PreviewSurfaceCfg(diffuse_color=(0.15, 0.25, 0.15)), + visible=True, + ), + }, + ) + test_marker = VisualizationMarkers(config) + test_marker.visualize( + translations=torch.tensor([[0.0, 0.0, 0.0], [0.2, 0.0, 0.0]], device=sim.device), + marker_indices=torch.tensor([0, 1], device=sim.device), + ) + + stage = sim_utils.get_current_stage() + instancer_prim = stage.GetPrimAtPath(test_marker.prim_path) + instancer = UsdGeom.PointInstancer(instancer_prim) + + assert instancer_prim.IsValid() + assert instancer + assert UsdGeom.Imageable(instancer_prim).GetVisibilityAttr().Get() != UsdGeom.Tokens.invisible + assert len(instancer.GetPositionsAttr().Get()) == 2 + assert list(instancer.GetProtoIndicesAttr().Get()) == [0, 1] + + def test_usd_marker(sim): """Test with marker from a USD.""" # create a marker @@ -80,29 +170,6 @@ def test_usd_marker(sim): assert test_marker.count == num_frames -def test_usd_marker_color(sim): - """Test with marker from a USD with its color modified.""" - # create a marker - config = FRAME_MARKER_CFG.copy() - config.prim_path = "/World/Visuals/test_frames" - config.markers["frame"].visual_material = sim_utils.PreviewSurfaceCfg(diffuse_color=(1.0, 0.0, 0.0)) - test_marker = VisualizationMarkers(config) - - # play the simulation - sim.reset() - # run with randomization of poses - for count in range(1000): - # sample random poses - if count % 50 == 0: - num_frames = torch.randint(10, 1000, (1,)).item() - frame_translations = torch.randn(num_frames, 3, device=sim.device) - frame_rotations = random_orientation(num_frames, device=sim.device) - # set the marker - test_marker.visualize(translations=frame_translations, orientations=frame_rotations) - # update the kit - sim.step() - - def test_multiple_prototypes_marker(sim): """Test with multiple prototypes of spheres.""" # create a marker @@ -126,9 +193,8 @@ def test_multiple_prototypes_marker(sim): sim.step() -@pytest.mark.flaky(max_runs=3, min_passes=1) -def test_visualization_time_based_on_prototypes(sim): - """Test with time taken when number of prototypes is increased.""" +def test_visualization_skips_updates_when_invisible(sim): + """When invisible, visualize should not update marker state.""" # create a marker config = POSITION_GOAL_MARKER_CFG.copy() config.prim_path = "/World/Visuals/test_protos" @@ -136,69 +202,408 @@ def test_visualization_time_based_on_prototypes(sim): # play the simulation sim.reset() - # number of frames - num_frames = 4096 # check that visibility is true assert test_marker.is_visible() - # run with randomization of poses and indices - frame_translations = torch.randn(num_frames, 3, device=sim.device) - marker_indices = torch.randint(0, test_marker.num_prototypes, (num_frames,), device=sim.device) - # set the marker - with Timer("Marker visualization with explicit indices") as timer: - test_marker.visualize(translations=frame_translations, marker_indices=marker_indices) - # save the time - time_with_marker_indices = timer.time_elapsed - - with Timer("Marker visualization with no indices") as timer: - test_marker.visualize(translations=frame_translations) - # save the time - time_with_no_marker_indices = timer.time_elapsed + frame_translations = torch.randn(4, 3, device=sim.device) + marker_indices = torch.zeros(4, dtype=torch.int32, device=sim.device) + test_marker.visualize(translations=frame_translations, marker_indices=marker_indices) + assert test_marker.count == 4 # update the kit sim.step() - # check that the time is less - assert time_with_no_marker_indices < time_with_marker_indices + # make invisible + test_marker.set_visibility(False) + # check that visibility is false + assert not test_marker.is_visible() + test_marker.visualize( + translations=torch.randn(8, 3, device=sim.device), + marker_indices=torch.zeros(8, dtype=torch.int32, device=sim.device), + ) -def test_visualization_time_based_on_visibility(sim): - """Test with visibility of markers. When invisible, the visualize call should return.""" - # create a marker + assert test_marker.count == 4 + + +def test_newton_marker_backend_registers_and_updates_state_without_frame_capture(sim): + """Newton marker backend state should be registered and ready for Newton-family viewers.""" + sim._visualizers.append(_FakeMarkerVisualizer(pumps_app_update=False)) config = POSITION_GOAL_MARKER_CFG.copy() - config.prim_path = "/World/Visuals/test_protos" + config.prim_path = "/World/Visuals/newton_marker_state" test_marker = VisualizationMarkers(config) + translations = torch.arange(6, dtype=torch.float32, device=sim.device).reshape(2, 3) + marker_indices = torch.tensor([0, 0], device=sim.device) - # play the simulation - sim.reset() - # number of frames - num_frames = 4096 + test_marker.visualize(translations=translations, marker_indices=marker_indices) - # check that visibility is true - assert test_marker.is_visible() - # run with randomization of poses and indices - frame_translations = torch.randn(num_frames, 3, device=sim.device) - marker_indices = torch.randint(0, test_marker.num_prototypes, (num_frames,), device=sim.device) - # set the marker - with Timer("Marker visualization") as timer: - test_marker.visualize(translations=frame_translations, marker_indices=marker_indices) - # save the time - time_with_visualization = timer.time_elapsed + newton_backend = test_marker._backends[0] + assert isinstance(newton_backend, newton_markers.NewtonVisualizationMarkers) + assert sim.vis_marker_registry.get_groups()[newton_backend.group_id] is newton_backend + assert torch.equal(newton_backend.translations, translations) + assert torch.equal(newton_backend.marker_indices, marker_indices.to(dtype=torch.int32)) + assert newton_backend.count == 2 - # update the kit - sim.step() - # make invisible - test_marker.set_visibility(False) - # check that visibility is false - assert not test_marker.is_visible() - # run with randomization of poses and indices - frame_translations = torch.randn(num_frames, 3, device=sim.device) - marker_indices = torch.randint(0, test_marker.num_prototypes, (num_frames,), device=sim.device) - # set the marker - with Timer("Marker no visualization") as timer: - test_marker.visualize(translations=frame_translations, marker_indices=marker_indices) - # save the time - time_with_no_visualization = timer.time_elapsed - - # check that the time is less - assert time_with_no_visualization < time_with_visualization +def test_newton_visualizer_step_renders_markers(monkeypatch: pytest.MonkeyPatch): + """NewtonVisualizer.step should ask active Newton marker groups to render.""" + marker_calls = [] + + class _FakeViewer: + _update_frequency = 1 + + def __init__(self): + self.calls = [] + + def is_paused(self): + return False + + def begin_frame(self, sim_time): + self.calls.append(("begin_frame", sim_time)) + + def log_state(self, state): + self.calls.append(("log_state", state)) + + def end_frame(self): + self.calls.append(("end_frame",)) + + class _FakeNewtonManager: + @staticmethod + def get_state(scene_data_provider=None): + assert scene_data_provider == "provider" + return {"state": "ok"} + + @staticmethod + def get_num_envs() -> int: + return 4 + + def _fake_render_markers(viewer, visible_env_ids, num_envs): + marker_calls.append((viewer, visible_env_ids, num_envs)) + + import isaaclab_newton.physics as newton_physics + + monkeypatch.setattr(newton_physics, "NewtonManager", _FakeNewtonManager) + monkeypatch.setattr(newton_visualizer, "render_newton_visualization_markers", _fake_render_markers) + + viewer = _FakeViewer() + visualizer = newton_visualizer.NewtonVisualizer(NewtonVisualizerCfg(enable_markers=True)) + visualizer._is_initialized = True + visualizer._is_closed = False + visualizer._viewer = viewer + visualizer._scene_data_provider = "provider" + visualizer._resolved_visible_env_ids = [1, 3] + + visualizer.step(0.25) + + assert viewer.calls == [("begin_frame", pytest.approx(0.25)), ("log_state", {"state": "ok"}), ("end_frame",)] + assert marker_calls == [(viewer, [1, 3], 4)] + + +def test_viser_visualizer_marker_render_failure_does_not_interrupt_state_updates( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +): + """Viser marker failures should be logged without dropping body state frames.""" + marker_calls = [] + + class _FakeViewer: + def __init__(self): + self.calls = [] + + def begin_frame(self, sim_time: float) -> None: + self.calls.append(("begin_frame", sim_time)) + + def log_state(self, state) -> None: + self.calls.append(("log_state", state)) + + def end_frame(self) -> None: + self.calls.append(("end_frame",)) + + class _FakeProvider: + num_envs = 4 + usd_stage = None + + def get_camera_transforms(self): + return {} + + class _FakeNewtonManager: + @staticmethod + def get_model(): + return "dummy-model" + + @staticmethod + def get_state(scene_data_provider=None): + assert scene_data_provider is provider + return {"state": "ok"} + + @staticmethod + def get_num_envs() -> int: + return provider.num_envs + + def _fake_create_viewer(self, record_to_viser: str | None, metadata: dict | None = None): + self._viewer = viewer + + def _raise_marker_render(*args, **kwargs): + marker_calls.append((args, kwargs)) + raise RuntimeError("marker overlay failed") + + import isaaclab_newton.physics as newton_physics + + provider = _FakeProvider() + viewer = _FakeViewer() + monkeypatch.setattr(newton_physics, "NewtonManager", _FakeNewtonManager) + monkeypatch.setattr(viser_visualizer.ViserVisualizer, "_create_viewer", _fake_create_viewer) + monkeypatch.setattr(viser_visualizer, "render_newton_visualization_markers", _raise_marker_render) + + visualizer = viser_visualizer.ViserVisualizer(ViserVisualizerCfg()) + visualizer.initialize(provider) + + with caplog.at_level("WARNING"): + visualizer.step(0.25) + + assert marker_calls + assert viewer.calls == [("begin_frame", pytest.approx(0.25)), ("log_state", {"state": "ok"}), ("end_frame",)] + assert "Marker rendering failed; continuing body updates" in caplog.text + + +def test_rerun_visualizer_marker_failure_still_ends_frame(monkeypatch: pytest.MonkeyPatch): + """Rerun should close the frame even if marker rendering raises.""" + captured = {} + + class _FakeViewer: + def __init__(self): + self.calls = [] + + def is_paused(self): + return False + + def begin_frame(self, sim_time): + self.calls.append(("begin_frame", sim_time)) + + def log_state(self, state): + self.calls.append(("log_state", state)) + + def end_frame(self): + self.calls.append(("end_frame",)) + + class _FakeProvider: + def get_metadata(self) -> dict: + return {"num_envs": 4} + + def get_newton_state(self): + return {"ok": True} + + def get_camera_transforms(self): + return {} + + class _FakeNewtonManager: + @staticmethod + def get_model(): + return "dummy-model" + + @staticmethod + def get_state(scene_data_provider=None): + captured["state_provider"] = scene_data_provider + return {"ok": True} + + @staticmethod + def get_num_envs() -> int: + return 4 + + def _raise_marker_render(*args, **kwargs): + raise RuntimeError("marker render failed") + + import isaaclab_newton.physics as newton_physics + + monkeypatch.setattr(newton_physics, "NewtonManager", _FakeNewtonManager) + monkeypatch.setattr(rerun_visualizer, "render_newton_visualization_markers", _raise_marker_render) + + visualizer = rerun_visualizer.RerunVisualizer(RerunVisualizerCfg()) + viewer = _FakeViewer() + visualizer._is_initialized = True + visualizer._is_closed = False + visualizer._viewer = viewer + visualizer._scene_data_provider = _FakeProvider() + visualizer._resolved_visible_env_ids = None + + with pytest.raises(RuntimeError, match="marker render failed"): + visualizer.step(0.25) + + assert captured["state_provider"] is visualizer._scene_data_provider + assert [call[0] for call in viewer.calls] == ["begin_frame", "log_state", "end_frame"] + + +def test_newton_marker_mesh_registration_is_per_viewer(monkeypatch: pytest.MonkeyPatch): + marker = object.__new__(newton_markers.NewtonVisualizationMarkers) + marker._registered_meshes = set() + + class _FakeMesh: + vertices = np.zeros((1, 3), dtype=np.float32) + indices = np.zeros((3,), dtype=np.int32) + normals = np.zeros((0, 3), dtype=np.float32) + uvs = np.zeros((0, 2), dtype=np.float32) + + class _FakeViewer: + def __init__(self): + self.meshes = [] + + def log_mesh(self, name, vertices, indices, **kwargs): + self.meshes.append((name, vertices, indices, kwargs)) + + monkeypatch.setattr(newton_markers, "_create_mesh", lambda cfg: _FakeMesh()) + monkeypatch.setattr(newton_markers.wp, "array", lambda value, dtype=None: value) + + spec = newton_markers._NewtonMarkerSpec(renderer="mesh", mesh_type="box", mesh_params={"size": (1.0, 1.0, 1.0)}) + viewer_a = _FakeViewer() + viewer_b = _FakeViewer() + + marker._ensure_mesh_registered(viewer_a, "/Visuals/marker/meshes/arrow", spec) + marker._ensure_mesh_registered(viewer_a, "/Visuals/marker/meshes/arrow", spec) + marker._ensure_mesh_registered(viewer_b, "/Visuals/marker/meshes/arrow", spec) + + assert len(viewer_a.meshes) == 1 + assert len(viewer_b.meshes) == 1 + + +class _FakeNewtonMarkerMesh: + vertices = np.zeros((1, 3), dtype=np.float32) + indices = np.zeros((3,), dtype=np.int32) + normals = np.zeros((0, 3), dtype=np.float32) + uvs = np.zeros((0, 2), dtype=np.float32) + + +class _FakeNewtonMarkerViewer: + def __init__(self): + self.meshes = [] + self.instances = [] + self.lines = [] + + def log_mesh(self, name, vertices, indices, **kwargs): + self.meshes.append((name, vertices, indices, kwargs)) + + def log_instances(self, batch_name, mesh_name, xforms, scales, colors, materials, hidden=False): + self.instances.append( + { + "batch_name": batch_name, + "mesh_name": mesh_name, + "xforms": xforms, + "scales": scales, + "colors": colors, + "materials": materials, + "hidden": hidden, + } + ) + + def log_lines(self, batch_name, starts, ends, colors, width=None, hidden=False): + self.lines.append( + { + "batch_name": batch_name, + "starts": starts, + "ends": ends, + "colors": colors, + "width": width, + "hidden": hidden, + } + ) + + +def _make_newton_marker_for_render( + *, + marker_names: list[str], + translations: torch.Tensor, + marker_indices: torch.Tensor | None = None, + visible: bool = True, +): + marker = object.__new__(newton_markers.NewtonVisualizationMarkers) + marker_cfg_type = type("MarkerCfg", (), {"visual_material": None}) + marker.cfg = type("Cfg", (), {"markers": {name: marker_cfg_type() for name in marker_names}})() + marker.group_id = "/Visuals/marker::test" + marker.visible = visible + marker.translations = translations + marker.orientations = torch.tensor([[0.0, 0.0, 0.0, 1.0]], dtype=torch.float32).repeat(translations.shape[0], 1) + marker.scales = torch.ones((translations.shape[0], 3), dtype=torch.float32) + marker.marker_indices = marker_indices + marker.count = translations.shape[0] + marker._registered_meshes = set() + marker._warned_unsupported = set() + return marker + + +def _patch_newton_marker_render_deps(monkeypatch: pytest.MonkeyPatch) -> None: + specs = { + "arrow": newton_markers._NewtonMarkerSpec( + renderer="mesh", + mesh_type="box", + mesh_params={"size": (1.0, 1.0, 1.0)}, + color=(1.0, 1.0, 1.0), + texture=np.zeros((2, 2, 3), dtype=np.uint8), + ), + "sphere": newton_markers._NewtonMarkerSpec(renderer="mesh", mesh_type="sphere", mesh_params={"radius": 1.0}), + "frame": newton_markers._NewtonMarkerSpec(renderer="frame"), + } + + monkeypatch.setattr(newton_markers, "_create_mesh", lambda cfg: _FakeNewtonMarkerMesh()) + monkeypatch.setattr(newton_markers.wp, "array", lambda value, dtype=None: value) + monkeypatch.setattr(newton_markers, "_resolve_newton_marker_cfg", lambda name, marker_cfg, cfg: specs[name]) + + +def test_newton_marker_render_filters_visible_envs(monkeypatch: pytest.MonkeyPatch): + _patch_newton_marker_render_deps(monkeypatch) + translations = torch.arange(8, dtype=torch.float32).unsqueeze(1).repeat(1, 3) + marker = _make_newton_marker_for_render( + marker_names=["arrow"], + translations=translations, + marker_indices=torch.zeros(8, dtype=torch.int32), + ) + viewer = _FakeNewtonMarkerViewer() + + marker.render(viewer, visible_env_ids=[1, 3], num_envs=4) + + assert len(viewer.instances) == 1 + assert viewer.instances[0]["hidden"] is False + assert viewer.instances[0]["xforms"][:, 0].tolist() == [1.0, 3.0, 5.0, 7.0] + + +def test_newton_marker_render_routes_instances_by_prototype(monkeypatch: pytest.MonkeyPatch): + _patch_newton_marker_render_deps(monkeypatch) + translations = torch.arange(4, dtype=torch.float32).unsqueeze(1).repeat(1, 3) + marker = _make_newton_marker_for_render( + marker_names=["arrow", "sphere"], + translations=translations, + marker_indices=torch.tensor([0, 1, 0, 1], dtype=torch.int32), + ) + viewer = _FakeNewtonMarkerViewer() + + marker.render(viewer, visible_env_ids=None, num_envs=4) + + visible_instances = [call for call in viewer.instances if not call["hidden"]] + assert [call["batch_name"] for call in visible_instances] == [ + "/Visuals/marker::test/arrow", + "/Visuals/marker::test/sphere", + ] + assert [call["xforms"].shape[0] for call in visible_instances] == [2, 2] + assert visible_instances[0]["materials"][:, 3].tolist() == [1.0, 1.0] + assert visible_instances[1]["materials"][:, 3].tolist() == [0.0, 0.0] + + +def test_newton_marker_render_hides_unselected_prototypes(monkeypatch: pytest.MonkeyPatch): + _patch_newton_marker_render_deps(monkeypatch) + marker = _make_newton_marker_for_render( + marker_names=["arrow", "sphere", "frame"], + translations=torch.zeros((3, 3), dtype=torch.float32), + marker_indices=torch.zeros(3, dtype=torch.int32), + ) + viewer = _FakeNewtonMarkerViewer() + + marker.render(viewer, visible_env_ids=None, num_envs=3) + + hidden_instances = [call for call in viewer.instances if call["hidden"]] + assert [call["batch_name"] for call in hidden_instances] == ["/Visuals/marker::test/sphere"] + assert viewer.lines == [ + { + "batch_name": "/Visuals/marker::test/frame", + "starts": None, + "ends": None, + "colors": None, + "width": None, + "hidden": True, + } + ] diff --git a/source/isaaclab/test/sim/test_deformable_backend_split.py b/source/isaaclab/test/sim/test_deformable_backend_split.py new file mode 100644 index 00000000000..dc170fe55c5 --- /dev/null +++ b/source/isaaclab/test/sim/test_deformable_backend_split.py @@ -0,0 +1,131 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Tests for the backend split of deformable schemas and materials.""" + +import dataclasses + +import isaaclab_physx.sim.schemas as physx_schemas +from isaaclab_newton.sim.schemas import NewtonDeformableBodyPropertiesCfg +from isaaclab_newton.sim.spawners.materials import ( + NewtonDeformableBodyMaterialCfg, + NewtonDeformableMaterialCfg, + NewtonSurfaceDeformableBodyMaterialCfg, +) +from isaaclab_physx.sim.schemas import ( + OmniPhysicsDeformableBodyPropertiesCfg, + PhysxDeformableBodyPropertiesCfg, + PhysxDeformableCollisionPropertiesCfg, +) +from isaaclab_physx.sim.schemas.schemas_cfg import PhysXDeformableBodyPropertiesCfg +from isaaclab_physx.sim.spawners.materials import ( + PhysxDeformableBodyMaterialCfg, + PhysXDeformableMaterialCfg, + PhysxSurfaceDeformableBodyMaterialCfg, +) + +import isaaclab.sim.schemas as schemas +import isaaclab.sim.spawners.materials.physics_materials_cfg as core_materials_cfg +from isaaclab.sim.schemas import DeformableBodyPropertiesBaseCfg +from isaaclab.sim.spawners.materials import ( + DeformableBodyMaterialBaseCfg, + SurfaceDeformableBodyMaterialBaseCfg, +) + + +def _field_names(cls) -> set[str]: + return {field.name for field in dataclasses.fields(cls)} + + +def _assert_no_property_prefix_field(cls): + assert "_property_prefix" not in _field_names(cls) + + +def test_common_deformable_property_cfg_has_no_backend_fields(): + """Common deformable properties are empty backend extension points.""" + fields = _field_names(DeformableBodyPropertiesBaseCfg) + + assert fields == set() + _assert_no_property_prefix_field(DeformableBodyPropertiesBaseCfg) + assert not hasattr(DeformableBodyPropertiesBaseCfg, "_usd_namespace") + assert not hasattr(DeformableBodyPropertiesBaseCfg, "_usd_applied_schema") + + +def test_common_deformable_material_cfg_has_no_backend_fields(): + """Common deformable material bases are empty backend extension points.""" + fields = _field_names(DeformableBodyMaterialBaseCfg) + surface_fields = _field_names(SurfaceDeformableBodyMaterialBaseCfg) + + assert fields == {"func"} + assert surface_fields == {"func"} + assert "DeformableBodyMaterialCfg" not in core_materials_cfg.__dict__ + assert "SurfaceDeformableBodyMaterialCfg" not in core_materials_cfg.__dict__ + assert not hasattr(core_materials_cfg, "PhysXDeformableMaterialCfg") + assert not hasattr(core_materials_cfg, "NewtonDeformableMaterialCfg") + assert not hasattr(DeformableBodyMaterialBaseCfg, "_usd_namespace") + assert not hasattr(DeformableBodyMaterialBaseCfg, "_usd_applied_schema") + assert not hasattr(SurfaceDeformableBodyMaterialBaseCfg, "_usd_namespace") + assert not hasattr(SurfaceDeformableBodyMaterialBaseCfg, "_usd_applied_schema") + _assert_no_property_prefix_field(DeformableBodyMaterialBaseCfg) + _assert_no_property_prefix_field(SurfaceDeformableBodyMaterialBaseCfg) + + +def test_physx_deformable_cfgs_use_core_schema_and_material_functions(): + """PhysX deformable cfgs own PhysX fields while schema and material functions stay in core.""" + props = PhysxDeformableBodyPropertiesCfg() + material = PhysxDeformableBodyMaterialCfg() + surface_material = PhysxSurfaceDeformableBodyMaterialCfg() + + assert not hasattr(props, "define_func") + assert not hasattr(props, "modify_func") + assert physx_schemas.define_deformable_body_properties is schemas.define_deformable_body_properties + assert physx_schemas.modify_deformable_body_properties is schemas.modify_deformable_body_properties + assert str(material.func) == "isaaclab.sim.spawners.materials.physics_materials:spawn_deformable_body_material" + assert str(surface_material.func) == str(material.func) + _assert_no_property_prefix_field(type(props)) + _assert_no_property_prefix_field(type(material)) + _assert_no_property_prefix_field(type(surface_material)) + assert PhysXDeformableBodyPropertiesCfg._usd_namespace == "physxDeformableBody" + assert PhysXDeformableBodyPropertiesCfg._usd_applied_schema == "PhysxBaseDeformableBodyAPI" + assert OmniPhysicsDeformableBodyPropertiesCfg._usd_namespace == "omniphysics" + assert OmniPhysicsDeformableBodyPropertiesCfg._usd_applied_schema is None + assert PhysxDeformableCollisionPropertiesCfg._usd_namespace == "physxCollision" + assert PhysxDeformableCollisionPropertiesCfg._usd_applied_schema == "PhysxCollisionAPI" + assert {"deformable_body_enabled", "kinematic_enabled", "mass"}.issubset(_field_names(type(props))) + assert {"density", "static_friction", "dynamic_friction", "youngs_modulus", "poissons_ratio"}.issubset( + _field_names(type(material)) + ) + assert "surface_thickness" in _field_names(type(surface_material)) + assert "bend_damping" in _field_names(type(surface_material)) + assert PhysXDeformableMaterialCfg._usd_namespace == "physxDeformableMaterial" + assert PhysXDeformableMaterialCfg._usd_applied_schema == "PhysxDeformableMaterialAPI" + assert "_usd_applied_schema" not in type(material).__dict__ + assert type(surface_material)._usd_namespace == "physxDeformableMaterial" + assert type(surface_material)._usd_applied_schema == "PhysxSurfaceDeformableMaterialAPI" + + +def test_newton_deformable_cfgs_use_core_schema_and_material_functions(): + """Newton deformable cfgs own Newton fields while schema and material functions stay in core.""" + props = NewtonDeformableBodyPropertiesCfg() + material = NewtonDeformableBodyMaterialCfg() + surface_material = NewtonSurfaceDeformableBodyMaterialCfg() + + assert not hasattr(props, "define_func") + assert not hasattr(props, "modify_func") + assert NewtonDeformableBodyPropertiesCfg._usd_namespace == "newton" + assert NewtonDeformableBodyPropertiesCfg._usd_applied_schema is None + assert str(material.func) == "isaaclab.sim.spawners.materials.physics_materials:spawn_deformable_body_material" + assert str(surface_material.func) == str(material.func) + _assert_no_property_prefix_field(type(props)) + _assert_no_property_prefix_field(type(material)) + _assert_no_property_prefix_field(type(surface_material)) + assert "deformable_body_enabled" not in _field_names(type(props)) + assert "kinematic_enabled" not in _field_names(type(props)) + assert "mass" not in _field_names(type(props)) + assert "youngs_modulus" not in _field_names(type(material)) + assert "poissons_ratio" not in _field_names(type(material)) + assert {"density", "particle_radius", "k_mu", "k_lambda", "k_damp"}.issubset(_field_names(type(material))) + assert NewtonDeformableMaterialCfg._usd_namespace == "newton" + assert NewtonDeformableMaterialCfg._usd_applied_schema is None diff --git a/source/isaaclab/test/sim/test_newton_manager_visualization_state.py b/source/isaaclab/test/sim/test_newton_manager_visualization_state.py index de0515c57e0..d444799620f 100644 --- a/source/isaaclab/test/sim/test_newton_manager_visualization_state.py +++ b/source/isaaclab/test/sim/test_newton_manager_visualization_state.py @@ -11,7 +11,7 @@ :meth:`NewtonManager._build_visualization_model_from_stage`), and :meth:`NewtonManager.update_visualization_state` must copy fresh transforms into ``_state_0.body_q`` via the new -:class:`~isaaclab.scene.scene_data_provider.SceneDataProvider`. +:class:`~isaaclab.scene_data.SceneDataProvider`. """ from __future__ import annotations @@ -26,8 +26,8 @@ def _reset_newton_manager_state(): NewtonManager._model = None NewtonManager._state_0 = None NewtonManager._num_envs = None - NewtonManager._physx_visualization_scene_data = None - NewtonManager._physx_visualization_mapping = None + NewtonManager._visualization_scene_data = None + NewtonManager._visualization_mapping = None def test_ensure_visualization_model_noop_when_backend_is_newton(monkeypatch): @@ -35,7 +35,7 @@ def test_ensure_visualization_model_noop_when_backend_is_newton(monkeypatch): from isaaclab_newton.physics import NewtonManager _reset_newton_manager_state() - monkeypatch.setattr(NewtonManager, "_backend_is_newton", classmethod(lambda cls: True)) + monkeypatch.setattr(NewtonManager, "_backend_is_newton", classmethod(lambda cls, scene_data_provider=None: True)) NewtonManager._ensure_visualization_model() assert NewtonManager._model is None assert NewtonManager._state_0 is None @@ -47,7 +47,7 @@ def test_ensure_visualization_model_builds_from_stage_when_backend_is_physx(monk from isaaclab_newton.physics import newton_manager as nm _reset_newton_manager_state() - monkeypatch.setattr(NewtonManager, "_backend_is_newton", classmethod(lambda cls: False)) + monkeypatch.setattr(NewtonManager, "_backend_is_newton", classmethod(lambda cls, scene_data_provider=None: False)) monkeypatch.setattr(nm, "get_current_stage", lambda *args, **kwargs: SimpleNamespace()) monkeypatch.setattr(nm, "replace_newton_shape_colors", lambda model, *a, **kw: 0) @@ -80,7 +80,7 @@ def test_ensure_visualization_model_empty_builder_logs_and_skips(monkeypatch, ca from isaaclab_newton.physics import newton_manager as nm _reset_newton_manager_state() - monkeypatch.setattr(NewtonManager, "_backend_is_newton", classmethod(lambda cls: False)) + monkeypatch.setattr(NewtonManager, "_backend_is_newton", classmethod(lambda cls, scene_data_provider=None: False)) monkeypatch.setattr(nm, "get_current_stage", lambda *args, **kwargs: SimpleNamespace()) class _EmptyBuilder: @@ -106,7 +106,7 @@ def test_ensure_visualization_model_populates_num_envs_when_backend_is_physx(mon from isaaclab_newton.physics import newton_manager as nm _reset_newton_manager_state() - monkeypatch.setattr(NewtonManager, "_backend_is_newton", classmethod(lambda cls: False)) + monkeypatch.setattr(NewtonManager, "_backend_is_newton", classmethod(lambda cls, scene_data_provider=None: False)) monkeypatch.setattr(nm, "get_current_stage", lambda *args, **kwargs: SimpleNamespace()) monkeypatch.setattr(nm, "replace_newton_shape_colors", lambda model, *a, **kw: 0) @@ -136,7 +136,7 @@ def test_ensure_visualization_model_missing_stage_leaves_state_unset(monkeypatch from isaaclab_newton.physics import newton_manager as nm _reset_newton_manager_state() - monkeypatch.setattr(NewtonManager, "_backend_is_newton", classmethod(lambda cls: False)) + monkeypatch.setattr(NewtonManager, "_backend_is_newton", classmethod(lambda cls, scene_data_provider=None: False)) monkeypatch.setattr(nm, "get_current_stage", lambda *args, **kwargs: None) with caplog.at_level("ERROR"): @@ -152,7 +152,7 @@ def test_update_visualization_state_noop_when_backend_is_newton(monkeypatch): from isaaclab_newton.physics import NewtonManager _reset_newton_manager_state() - monkeypatch.setattr(NewtonManager, "_backend_is_newton", classmethod(lambda cls: True)) + monkeypatch.setattr(NewtonManager, "_backend_is_newton", classmethod(lambda cls, scene_data_provider=None: True)) # Pre-set sentinel values to ensure update doesn't touch them. NewtonManager._model = "live-model" diff --git a/source/isaaclab/test/sim/test_schemas.py b/source/isaaclab/test/sim/test_schemas.py index 1fb03ede143..32143a8bb43 100644 --- a/source/isaaclab/test/sim/test_schemas.py +++ b/source/isaaclab/test/sim/test_schemas.py @@ -28,9 +28,6 @@ PhysxJointDrivePropertiesCfg, PhysxRigidBodyPropertiesCfg, ) -from isaaclab_physx.sim.schemas import ( - PhysXCollisionPropertiesCfg as PhysxDeformableCollisionAliasCfg, -) from isaaclab_physx.sim.spawners.materials import PhysxRigidBodyMaterialCfg, RigidBodyMaterialCfg from pxr import UsdPhysics @@ -418,19 +415,6 @@ def test_collision_deprecation_alias(setup_simulation): assert "5.0" in str(deprecations[0].message) -@pytest.mark.isaacsim_ci -def test_physx_capitalx_collision_deprecation_alias(setup_simulation): - """Instantiating the legacy ``PhysXCollisionPropertiesCfg`` (capital X, deformable) - name emits exactly one ``DeprecationWarning`` pointing to - ``PhysxDeformableCollisionPropertiesCfg``.""" - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - PhysxDeformableCollisionAliasCfg() - deprecations = [w for w in caught if issubclass(w.category, DeprecationWarning)] - assert len(deprecations) == 1, f"expected exactly one DeprecationWarning, got {len(deprecations)}" - assert "PhysxDeformableCollisionPropertiesCfg" in str(deprecations[0].message) - - @pytest.mark.isaacsim_ci def test_articulation_root_base_cfg_writes_articulation_enabled(setup_simulation): """Setting ``articulation_enabled`` on the base ``ArticulationRootBaseCfg`` must author diff --git a/source/isaaclab/test/sim/test_schemas_shim.py b/source/isaaclab/test/sim/test_schemas_shim.py index 11e6e5d1ba2..9a9c510e09b 100644 --- a/source/isaaclab/test/sim/test_schemas_shim.py +++ b/source/isaaclab/test/sim/test_schemas_shim.py @@ -29,8 +29,9 @@ "PhysxJointDrivePropertiesCfg", "CollisionPropertiesCfg", "PhysxCollisionPropertiesCfg", + "DeformableBodyPropertiesCfg", + "PhysxDeformableBodyPropertiesCfg", "PhysxDeformableCollisionPropertiesCfg", - "PhysXCollisionPropertiesCfg", "ArticulationRootPropertiesCfg", "PhysxArticulationRootPropertiesCfg", "MeshCollisionPropertiesCfg", @@ -54,7 +55,7 @@ "RigidBodyPropertiesCfg", "JointDrivePropertiesCfg", "CollisionPropertiesCfg", - "PhysXCollisionPropertiesCfg", + "DeformableBodyPropertiesCfg", "ArticulationRootPropertiesCfg", "MeshCollisionPropertiesCfg", "ConvexHullPropertiesCfg", @@ -67,8 +68,18 @@ ] FORWARDED_MATERIAL_NAMES = [ + "DeformableBodyMaterialCfg", "RigidBodyMaterialCfg", + "SurfaceDeformableBodyMaterialCfg", "PhysxRigidBodyMaterialCfg", + "PhysxDeformableBodyMaterialCfg", + "PhysxSurfaceDeformableBodyMaterialCfg", +] + +DEPRECATED_FORWARDED_MATERIAL_NAMES = [ + "DeformableBodyMaterialCfg", + "RigidBodyMaterialCfg", + "SurfaceDeformableBodyMaterialCfg", ] @@ -132,13 +143,14 @@ def test_deprecated_aliases_emit_deprecation_warning(name): assert len(deprecations) == 1, f"{name}: expected one DeprecationWarning, got {len(deprecations)}" -def test_deprecated_material_alias_emits_deprecation_warning(): - """Instantiating ``RigidBodyMaterialCfg`` via the shim still emits ``DeprecationWarning``.""" +@pytest.mark.parametrize("name", DEPRECATED_FORWARDED_MATERIAL_NAMES) +def test_deprecated_material_aliases_emit_deprecation_warning(name): + """Instantiating a deprecated material alias via the shim still emits ``DeprecationWarning``.""" with warnings.catch_warnings(record=True) as caught: warnings.simplefilter("always") - materials.RigidBodyMaterialCfg() + getattr(materials, name)() deprecations = [w for w in caught if issubclass(w.category, DeprecationWarning)] - assert len(deprecations) == 1 + assert len(deprecations) == 1, f"{name}: expected one DeprecationWarning, got {len(deprecations)}" assert "5.0" in str(deprecations[0].message) @@ -158,6 +170,13 @@ def test_new_material_class_does_not_emit_deprecation_warning(): assert not any(issubclass(w.category, DeprecationWarning) for w in caught) +def test_deformable_component_cfg_is_not_forwarded_from_core(): + """Component deformable cfgs are backend-owned and not forwarded from ``isaaclab``.""" + assert not hasattr(schemas, "PhysXDeformableBodyPropertiesCfg") + assert not hasattr(sim_utils, "PhysXDeformableBodyPropertiesCfg") + assert not hasattr(schemas_cfg_submodule, "PhysXDeformableBodyPropertiesCfg") + + def test_dir_lists_forwarded_names(): """``dir(isaaclab.sim.schemas)`` includes the forwarded names so IDE autocomplete works.""" listing = dir(schemas) diff --git a/source/isaaclab/test/sim/test_simulation_context_visualizers.py b/source/isaaclab/test/sim/test_simulation_context_visualizers.py index cf136c37053..094975033ab 100644 --- a/source/isaaclab/test/sim/test_simulation_context_visualizers.py +++ b/source/isaaclab/test/sim/test_simulation_context_visualizers.py @@ -11,18 +11,21 @@ from typing import Any, cast import isaaclab_visualizers.kit.kit_visualizer as kit_visualizer -import isaaclab_visualizers.newton.newton_visualization_markers as newton_markers import isaaclab_visualizers.rerun.rerun_visualizer as rerun_visualizer import isaaclab_visualizers.viser.viser_visualizer as viser_visualizer -import numpy as np import pytest -import torch from isaaclab_visualizers.kit.kit_visualizer_cfg import KitVisualizerCfg +from isaaclab_visualizers.newton.newton_visualizer_cfg import NewtonVisualizerCfg from isaaclab_visualizers.rerun.rerun_visualizer_cfg import RerunVisualizerCfg from isaaclab_visualizers.viser.viser_visualizer_cfg import ViserVisualizerCfg -from isaaclab.markers.vis_marker_registry import VisMarkerRegistry from isaaclab.sim.simulation_context import SimulationContext +from isaaclab.visualizers.visualizer_cfg import VisualizerCfg + + +def test_web_visualizer_cfgs_do_not_open_browser_by_default(): + assert RerunVisualizerCfg().open_browser is False + assert ViserVisualizerCfg().open_browser is False class _FakePhysicsManager: @@ -114,6 +117,9 @@ def pumps_app_update(self): def supports_markers(self): return False + def flush_startup_messages(self): + pass + def _make_context(visualizers, provider=None): ctx = object.__new__(SimulationContext) @@ -188,26 +194,6 @@ def test_update_visualizers_handles_training_pause_loop(): assert viz.step_calls == [0.0, 0.2] -def test_vis_marker_registry_dispatch_allows_callback_mutation(): - registry = VisMarkerRegistry() - calls = [] - - def _remove_other_callback(event): - calls.append(("remove_other", event)) - registry.remove_callback("other") - - def _other_callback(event): - calls.append(("other", event)) - - registry.add_callback("remove_other", _remove_other_callback) - registry.add_callback("other", _other_callback) - - registry.dispatch_callbacks("tick") - - assert calls == [("remove_other", "tick"), ("other", "tick")] - assert "other" not in registry._callbacks - - class _DummyViserSceneDataProvider: @property def num_envs(self) -> int: @@ -249,7 +235,7 @@ def _fake_create_viewer(self, record_to_viser: str | None, metadata: dict | None monkeypatch.setattr(viser_visualizer.ViserVisualizer, "_create_viewer", _fake_create_viewer) - state_calls: list[None] = [] + state_calls: list[object] = [] class _FakeNewtonManager: @staticmethod @@ -257,8 +243,8 @@ def get_model(): return "dummy-model" @staticmethod - def get_state(): - state_calls.append(None) + def get_state(scene_data_provider=None): + state_calls.append(scene_data_provider) return {"state_call": len(state_calls)} @staticmethod @@ -274,241 +260,13 @@ def get_num_envs() -> int: visualizer.step(0.25) assert visualizer.is_initialized - assert state_calls == [None, None] + assert state_calls == [provider, provider] assert visualizer._sim_time == pytest.approx(0.25) assert viewer.calls[0][0] == "begin_frame" assert viewer.calls[0][1] == pytest.approx(0.25) - # log_state passes NewtonManager.get_state() through as-is; no env_ids merged in. - assert viewer.calls[1] == ("log_state", {"state_call": 2}) - assert viewer.calls[2] == ("end_frame",) - - -def test_viser_visualizer_marker_render_failure_does_not_interrupt_state_updates( - monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture -): - provider = _DummyViserSceneDataProvider() - viewer = _DummyViserViewer() - marker_calls = [] - - def _fake_create_viewer(self, record_to_viser: str | None, metadata: dict | None = None): - self._viewer = viewer - - def _raise_marker_render(*args, **kwargs): - marker_calls.append((args, kwargs)) - raise RuntimeError("marker overlay failed") - - monkeypatch.setattr(viser_visualizer.ViserVisualizer, "_create_viewer", _fake_create_viewer) - monkeypatch.setattr(viser_visualizer, "render_newton_visualization_markers", _raise_marker_render) - - state_calls: list[None] = [] - - class _FakeNewtonManager: - @staticmethod - def get_model(): - return "dummy-model" - - @staticmethod - def get_state(): - state_calls.append(None) - return {"state_call": len(state_calls)} - - @staticmethod - def get_num_envs() -> int: - return 1 - - import isaaclab_newton.physics as _np_mod - - monkeypatch.setattr(_np_mod, "NewtonManager", _FakeNewtonManager) - - visualizer = viser_visualizer.ViserVisualizer(ViserVisualizerCfg()) - visualizer.initialize(cast(Any, provider)) - - with caplog.at_level("WARNING"): - visualizer.step(0.25) - - assert marker_calls - assert viewer.calls[0][0] == "begin_frame" + # log_state passes NewtonManager.get_state(provider) through as-is; no env_ids merged in. assert viewer.calls[1] == ("log_state", {"state_call": 2}) assert viewer.calls[2] == ("end_frame",) - assert "Marker rendering failed; continuing body updates" in caplog.text - - -def test_newton_marker_mesh_registration_is_per_viewer(monkeypatch: pytest.MonkeyPatch): - marker = object.__new__(newton_markers.NewtonVisualizationMarkers) - marker._registered_meshes = set() - - class _FakeMesh: - vertices = np.zeros((1, 3), dtype=np.float32) - indices = np.zeros((3,), dtype=np.int32) - normals = np.zeros((0, 3), dtype=np.float32) - uvs = np.zeros((0, 2), dtype=np.float32) - - class _FakeViewer: - def __init__(self): - self.meshes = [] - - def log_mesh(self, name, vertices, indices, **kwargs): - self.meshes.append((name, vertices, indices, kwargs)) - - monkeypatch.setattr(newton_markers, "_create_mesh", lambda cfg: _FakeMesh()) - monkeypatch.setattr(newton_markers.wp, "array", lambda value, dtype=None: value) - - spec = newton_markers._NewtonMarkerSpec(renderer="mesh", mesh_type="box", mesh_params={"size": (1.0, 1.0, 1.0)}) - viewer_a = _FakeViewer() - viewer_b = _FakeViewer() - - marker._ensure_mesh_registered(viewer_a, "/Visuals/marker/meshes/arrow", spec) - marker._ensure_mesh_registered(viewer_a, "/Visuals/marker/meshes/arrow", spec) - marker._ensure_mesh_registered(viewer_b, "/Visuals/marker/meshes/arrow", spec) - - assert len(viewer_a.meshes) == 1 - assert len(viewer_b.meshes) == 1 - - -class _FakeNewtonMarkerMesh: - vertices = np.zeros((1, 3), dtype=np.float32) - indices = np.zeros((3,), dtype=np.int32) - normals = np.zeros((0, 3), dtype=np.float32) - uvs = np.zeros((0, 2), dtype=np.float32) - - -class _FakeNewtonMarkerViewer: - def __init__(self): - self.meshes = [] - self.instances = [] - self.lines = [] - - def log_mesh(self, name, vertices, indices, **kwargs): - self.meshes.append((name, vertices, indices, kwargs)) - - def log_instances(self, batch_name, mesh_name, xforms, scales, colors, materials, hidden=False): - self.instances.append( - { - "batch_name": batch_name, - "mesh_name": mesh_name, - "xforms": xforms, - "scales": scales, - "colors": colors, - "materials": materials, - "hidden": hidden, - } - ) - - def log_lines(self, batch_name, starts, ends, colors, width=None, hidden=False): - self.lines.append( - { - "batch_name": batch_name, - "starts": starts, - "ends": ends, - "colors": colors, - "width": width, - "hidden": hidden, - } - ) - - -def _make_newton_marker_for_render( - *, - marker_names: list[str], - translations: torch.Tensor, - marker_indices: torch.Tensor | None = None, - visible: bool = True, -): - marker = object.__new__(newton_markers.NewtonVisualizationMarkers) - marker_cfg_type = type("MarkerCfg", (), {"visual_material": None}) - marker.cfg = type("Cfg", (), {"markers": {name: marker_cfg_type() for name in marker_names}})() - marker.group_id = "/Visuals/marker::test" - marker.visible = visible - marker.translations = translations - marker.orientations = torch.tensor([[0.0, 0.0, 0.0, 1.0]], dtype=torch.float32).repeat(translations.shape[0], 1) - marker.scales = torch.ones((translations.shape[0], 3), dtype=torch.float32) - marker.marker_indices = marker_indices - marker.count = translations.shape[0] - marker._registered_meshes = set() - marker._warned_unsupported = set() - return marker - - -def _patch_newton_marker_render_deps(monkeypatch: pytest.MonkeyPatch) -> None: - specs = { - "arrow": newton_markers._NewtonMarkerSpec( - renderer="mesh", - mesh_type="box", - mesh_params={"size": (1.0, 1.0, 1.0)}, - color=(1.0, 1.0, 1.0), - texture=np.zeros((2, 2, 3), dtype=np.uint8), - ), - "sphere": newton_markers._NewtonMarkerSpec(renderer="mesh", mesh_type="sphere", mesh_params={"radius": 1.0}), - "frame": newton_markers._NewtonMarkerSpec(renderer="frame"), - } - - monkeypatch.setattr(newton_markers, "_create_mesh", lambda cfg: _FakeNewtonMarkerMesh()) - monkeypatch.setattr(newton_markers.wp, "array", lambda value, dtype=None: value) - monkeypatch.setattr(newton_markers, "_resolve_newton_marker_cfg", lambda name, marker_cfg, cfg: specs[name]) - - -def test_newton_marker_render_filters_visible_envs(monkeypatch: pytest.MonkeyPatch): - _patch_newton_marker_render_deps(monkeypatch) - translations = torch.arange(8, dtype=torch.float32).unsqueeze(1).repeat(1, 3) - marker = _make_newton_marker_for_render( - marker_names=["arrow"], - translations=translations, - marker_indices=torch.zeros(8, dtype=torch.int32), - ) - viewer = _FakeNewtonMarkerViewer() - - marker.render(viewer, visible_env_ids=[1, 3], num_envs=4) - - assert len(viewer.instances) == 1 - assert viewer.instances[0]["hidden"] is False - assert viewer.instances[0]["xforms"][:, 0].tolist() == [1.0, 3.0, 5.0, 7.0] - - -def test_newton_marker_render_routes_instances_by_prototype(monkeypatch: pytest.MonkeyPatch): - _patch_newton_marker_render_deps(monkeypatch) - translations = torch.arange(4, dtype=torch.float32).unsqueeze(1).repeat(1, 3) - marker = _make_newton_marker_for_render( - marker_names=["arrow", "sphere"], - translations=translations, - marker_indices=torch.tensor([0, 1, 0, 1], dtype=torch.int32), - ) - viewer = _FakeNewtonMarkerViewer() - - marker.render(viewer, visible_env_ids=None, num_envs=4) - - visible_instances = [call for call in viewer.instances if not call["hidden"]] - assert [call["batch_name"] for call in visible_instances] == [ - "/Visuals/marker::test/arrow", - "/Visuals/marker::test/sphere", - ] - assert [call["xforms"].shape[0] for call in visible_instances] == [2, 2] - assert visible_instances[0]["materials"][:, 3].tolist() == [1.0, 1.0] - assert visible_instances[1]["materials"][:, 3].tolist() == [0.0, 0.0] - - -def test_newton_marker_render_hides_unselected_prototypes(monkeypatch: pytest.MonkeyPatch): - _patch_newton_marker_render_deps(monkeypatch) - marker = _make_newton_marker_for_render( - marker_names=["arrow", "sphere", "frame"], - translations=torch.zeros((3, 3), dtype=torch.float32), - marker_indices=torch.zeros(3, dtype=torch.int32), - ) - viewer = _FakeNewtonMarkerViewer() - - marker.render(viewer, visible_env_ids=None, num_envs=3) - - hidden_instances = [call for call in viewer.instances if call["hidden"]] - assert [call["batch_name"] for call in hidden_instances] == ["/Visuals/marker::test/sphere"] - assert viewer.lines == [ - { - "batch_name": "/Visuals/marker::test/frame", - "starts": None, - "ends": None, - "colors": None, - "width": None, - "hidden": True, - } - ] @pytest.mark.parametrize( @@ -531,6 +289,7 @@ def __init__( self, *, port: int, + bind_address: str, label: str | None, verbose: bool, share: bool, @@ -539,6 +298,7 @@ def __init__( ): captured["init"] = { "port": port, + "bind_address": bind_address, "label": label, "verbose": verbose, "share": share, @@ -555,6 +315,10 @@ def set_visible_worlds(self, worlds) -> None: def set_world_offsets(self, spacing) -> None: captured["set_world_offsets"] = tuple(spacing) + @property + def share_url(self) -> str | None: + return None + monkeypatch.setattr(viser_visualizer, "NewtonViewerViser", _FakeNewtonViewerViser) monkeypatch.setattr( viser_visualizer.ViserVisualizer, @@ -574,6 +338,7 @@ def set_world_offsets(self, spacing) -> None: visualizer._create_viewer(record_to_viser="record.viser", metadata={"num_envs": 8}) assert captured["set_model"] == "dummy-model" + assert captured["init"]["bind_address"] == cfg.bind_address assert captured["visible_worlds"] == expected_visible assert captured["set_world_offsets"] == (0.0, 0.0, 0.0) @@ -605,6 +370,7 @@ def __init__( keep_historical_data: bool, keep_scalar_history: bool, record_to_rrd: str | None, + open_browser: bool, ): captured["init"] = { "app_id": app_id, @@ -615,6 +381,7 @@ def __init__( "keep_historical_data": keep_historical_data, "keep_scalar_history": keep_scalar_history, "record_to_rrd": record_to_rrd, + "open_browser": open_browser, } def set_model(self, model: Any) -> None: @@ -647,7 +414,8 @@ def get_model(): return "dummy-model" @staticmethod - def get_state(): + def get_state(scene_data_provider=None): + captured["state_provider"] = scene_data_provider return {"ok": True} @staticmethod @@ -683,69 +451,6 @@ def get_num_envs() -> int: assert captured["set_world_offsets"] == (0.0, 0.0, 0.0) -def test_rerun_visualizer_marker_failure_still_ends_frame(monkeypatch: pytest.MonkeyPatch): - class _FakeRerunViewer: - def __init__(self): - self.calls = [] - - def is_paused(self): - return False - - def begin_frame(self, sim_time): - self.calls.append(("begin_frame", sim_time)) - - def log_state(self, state): - self.calls.append(("log_state", state)) - - def end_frame(self): - self.calls.append(("end_frame",)) - - class _DummyRerunSceneDataProvider: - def get_metadata(self) -> dict: - return {"num_envs": 4} - - def get_newton_state(self): - return {"ok": True} - - def get_camera_transforms(self): - return {} - - def _raise_marker_render(*args, **kwargs): - raise RuntimeError("marker render failed") - - monkeypatch.setattr(rerun_visualizer, "render_newton_visualization_markers", _raise_marker_render) - - class _FakeNewtonManager: - @staticmethod - def get_model(): - return "dummy-model" - - @staticmethod - def get_state(): - return {"ok": True} - - @staticmethod - def get_num_envs() -> int: - return 4 - - import isaaclab_newton.physics as _np_mod - - monkeypatch.setattr(_np_mod, "NewtonManager", _FakeNewtonManager) - - visualizer = rerun_visualizer.RerunVisualizer(RerunVisualizerCfg()) - viewer = _FakeRerunViewer() - visualizer._is_initialized = True - visualizer._is_closed = False - visualizer._viewer = viewer - visualizer._scene_data_provider = _DummyRerunSceneDataProvider() - visualizer._resolved_visible_env_ids = None - - with pytest.raises(RuntimeError, match="marker render failed"): - visualizer.step(0.25) - - assert [call[0] for call in viewer.calls] == ["begin_frame", "log_state", "end_frame"] - - def test_kit_visualizer_default_camera_source_does_not_require_camera_prim(monkeypatch: pytest.MonkeyPatch): """Default ``--viz kit`` should work for envs without a camera prim.""" @@ -800,12 +505,29 @@ def get_usd_stage(self): visualizer._setup_viewport() - assert cfg.cam_source == "cfg" + assert not cfg.tiled_cam_view assert applied_camera_poses == [(cfg.eye, cfg.lookat)] assert viewport_window.viewport_api.set_active_camera_calls == [] assert visualizer._controlled_camera_path == "/OmniverseKit_Persp" +def test_kit_visualizer_default_camera_source_accepts_set_camera_view(monkeypatch: pytest.MonkeyPatch): + """Default Kit visualizer camera follows SimulationContext/ViewportCameraController updates.""" + applied_camera_poses = [] + monkeypatch.setattr( + kit_visualizer.KitVisualizer, + "_set_viewport_camera", + lambda self, eye, target: applied_camera_poses.append((tuple(eye), tuple(target))), + ) + + visualizer = kit_visualizer.KitVisualizer(KitVisualizerCfg()) + visualizer._is_initialized = True + + visualizer.set_camera_view((1.0, 2.0, 3.0), (0.0, 0.0, 1.0)) + + assert applied_camera_poses == [((1.0, 2.0, 3.0), (0.0, 0.0, 1.0))] + + def test_get_cli_visualizer_types_handles_non_string_setting_without_crashing(): ctx = object.__new__(SimulationContext) ctx.get_setting = lambda name: {"types": "newton,kit"} if name == "/isaaclab/visualizer/types" else None @@ -840,6 +562,7 @@ def initialize(self, provider): def _make_context_with_settings( settings: dict, visualizer_cfgs=None, + default_visualizer_cfg=None, *, has_gui: bool = False, has_offscreen_render: bool = False, @@ -855,6 +578,7 @@ def _make_context_with_settings( (), { "visualizer_cfgs": visualizer_cfgs, + "default_visualizer_cfg": default_visualizer_cfg, "physics": type("PhysicsCfg", (), {"dt": 0.01})(), "dt": 0.01, "render_interval": 1, @@ -876,6 +600,27 @@ def _make_context_with_settings( return ctx +def test_default_visualizer_cfg_applies_to_cli_created_configs(): + settings = { + "/isaaclab/visualizer/types": "newton", + "/isaaclab/visualizer/explicit": True, + "/isaaclab/visualizer/disable_all": False, + "/isaaclab/visualizer/max_visible_envs": None, + } + default_cfg = VisualizerCfg( + tiled_cam_target_prim_path="/World/envs/*/Object", + tiled_cam_eye=(1.0, -1.0, 0.5), + ) + ctx = _make_context_with_settings(settings, default_visualizer_cfg=default_cfg) + + cfgs = ctx._resolve_visualizer_cfgs() + + assert len(cfgs) == 1 + assert isinstance(cfgs[0], NewtonVisualizerCfg) + assert cfgs[0].tiled_cam_target_prim_path == "/World/envs/*/Object" + assert cfgs[0].tiled_cam_eye == (1.0, -1.0, 0.5) + + def test_is_rendering_true_when_only_cfg_visualizer_is_set(): cfg_visualizer = type("CfgVisualizer", (), {"visualizer_type": "newton"})() settings = { diff --git a/source/isaaclab/test/sim/test_spawn_meshes.py b/source/isaaclab/test/sim/test_spawn_meshes.py index 7b4c4678ca5..97aca4a23f7 100644 --- a/source/isaaclab/test/sim/test_spawn_meshes.py +++ b/source/isaaclab/test/sim/test_spawn_meshes.py @@ -120,18 +120,19 @@ def test_spawn_sphere(sim): @pytest.mark.parametrize("resolution", [(1, 1), (3, 2)]) -def test_spawn_square(sim, resolution): - """Test spawning of UsdGeomMesh as a square prim.""" - # Spawn square - cfg = sim_utils.MeshSquareCfg(size=1.0, resolution=resolution) - prim = cfg.func("/World/Square", cfg) +@pytest.mark.parametrize("size", [(1.0, 1.0), (1.5, 0.8)]) +def test_spawn_rectangle(sim, resolution, size): + """Test spawning of UsdGeomMesh as a rectangle prim.""" + # Spawn rectangle + cfg = sim_utils.MeshRectangleCfg(size=size, resolution=resolution) + prim = cfg.func("/World/Rectangle", cfg) # Check validity assert prim.IsValid() - assert sim.stage.GetPrimAtPath("/World/Square").IsValid() + assert sim.stage.GetPrimAtPath("/World/Rectangle").IsValid() assert prim.GetPrimTypeInfo().GetTypeName() == "Xform" # Check properties - prim = sim.stage.GetPrimAtPath("/World/Square/geometry/mesh") + prim = sim.stage.GetPrimAtPath("/World/Rectangle/geometry/mesh") assert prim.GetPrimTypeInfo().GetTypeName() == "Mesh" diff --git a/source/isaaclab/test/visualizers/test_visualizer.py b/source/isaaclab/test/visualizers/test_visualizer.py index 4b050431eb8..3cc8deed56b 100644 --- a/source/isaaclab/test/visualizers/test_visualizer.py +++ b/source/isaaclab/test/visualizers/test_visualizer.py @@ -11,7 +11,9 @@ from types import SimpleNamespace import pytest +import torch +from isaaclab.envs.utils.camera_view import apply_camera_view_from_origins from isaaclab.visualizers.base_visualizer import BaseVisualizer from isaaclab.visualizers.visualizer import Visualizer from isaaclab.visualizers.visualizer_cfg import VisualizerCfg @@ -27,6 +29,13 @@ def test_create_visualizer_raises_for_base_cfg(): cfg.create_visualizer() +def test_visualizer_cfg_tiled_camera_view_is_opt_in(): + cfg = VisualizerCfg() + assert cfg.focal_length == 12.0 + assert cfg.tiled_cam_view is False + assert cfg.tiled_cam_num == 16 + + def test_create_visualizer_raises_for_unknown_type(): cfg = VisualizerCfg(visualizer_type="unknown-backend") with pytest.raises(ValueError, match="not registered"): @@ -90,6 +99,33 @@ def get_camera_transforms(self): return self._transforms +class _FakeCamera: + device = "cpu" + + def __init__(self): + self.set_world_poses_from_view_calls = [] + self.update_poses_calls = [] + + def set_world_poses_from_view(self, eyes, targets, env_ids=None): + self.set_world_poses_from_view_calls.append((eyes.clone(), targets.clone(), env_ids)) + + def _update_poses(self, dt): + self.update_poses_calls.append(dt) + + +def test_apply_camera_view_from_origins_forwards_env_ids(): + camera = _FakeCamera() + origins = torch.tensor([[1.0, 2.0, 3.0]]) + + apply_camera_view_from_origins(camera, origins, eye=(0.5, 0.0, 1.0), lookat=(0.0, 0.0, 0.0), env_ids=[2]) + + eyes, targets, env_ids = camera.set_world_poses_from_view_calls[0] + assert eyes.tolist() == [[1.5, 2.0, 4.0]] + assert targets.tolist() == [[1.0, 2.0, 3.0]] + assert env_ids == [2] + assert camera.update_poses_calls == [None] + + def test_compute_visualized_env_ids_cap_only_returns_none(): """Cap-only path: :meth:`_compute_visualized_env_ids` is ``None``. diff --git a/source/isaaclab_contrib/changelog.d/mym-deformable-backend-split.skip b/source/isaaclab_contrib/changelog.d/mym-deformable-backend-split.skip new file mode 100644 index 00000000000..37d59e91ab7 --- /dev/null +++ b/source/isaaclab_contrib/changelog.d/mym-deformable-backend-split.skip @@ -0,0 +1 @@ +Test-only updates for backend-specific deformable cfg imports. diff --git a/source/isaaclab_contrib/changelog.d/mym-deformable_experimental.minor.rst b/source/isaaclab_contrib/changelog.d/mym-deformable_experimental.minor.rst new file mode 100644 index 00000000000..4167f0a3332 --- /dev/null +++ b/source/isaaclab_contrib/changelog.d/mym-deformable_experimental.minor.rst @@ -0,0 +1,16 @@ +Added +^^^^^ + +* Added :mod:`isaaclab_contrib.deformable` with contributed Newton deformable + asset and VBD solver support, including + :class:`~isaaclab_contrib.deformable.DeformableObject`, + :class:`~isaaclab_contrib.deformable.VBDSolverCfg`, + :class:`~isaaclab_contrib.deformable.CoupledMJWarpVBDSolverCfg`, and + :class:`~isaaclab_contrib.deformable.CoupledFeatherstoneVBDSolverCfg` for + one- and two-way rigid-deformable coupling. +* Added :class:`~isaaclab_contrib.deformable.NewtonModelCfg` for shared Newton + deformable contact parameters. +* Added Newton deformable coupling documentation with Franka soft-body lift + tuning guidance for + :class:`~isaaclab_contrib.deformable.CoupledMJWarpVBDSolverCfg` and + :class:`~isaaclab_contrib.deformable.NewtonModelCfg`. diff --git a/source/isaaclab_contrib/isaaclab_contrib/deformable/__init__.py b/source/isaaclab_contrib/isaaclab_contrib/deformable/__init__.py new file mode 100644 index 00000000000..02c5059784e --- /dev/null +++ b/source/isaaclab_contrib/isaaclab_contrib/deformable/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Sub-package for externally contributed assets. + +This package contains contributed code that depends on Isaac Lab's public API but is not required for core functionality. This includes implementations of Newton solvers for deformables. +""" + +from isaaclab.utils.module import lazy_export + +lazy_export() diff --git a/source/isaaclab_contrib/isaaclab_contrib/deformable/__init__.pyi b/source/isaaclab_contrib/isaaclab_contrib/deformable/__init__.pyi new file mode 100644 index 00000000000..b2438a247a4 --- /dev/null +++ b/source/isaaclab_contrib/isaaclab_contrib/deformable/__init__.pyi @@ -0,0 +1,22 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +__all__ = [ + "CoupledFeatherstoneVBDSolverCfg", + "CoupledMJWarpVBDSolverCfg", + "DeformableObject", + "DeformableObjectData", + "NewtonModelCfg", + "VBDSolverCfg", +] + +from .deformable_object import DeformableObject +from .deformable_object_data import DeformableObjectData +from .newton_manager_cfg import ( + CoupledFeatherstoneVBDSolverCfg, + CoupledMJWarpVBDSolverCfg, + NewtonModelCfg, + VBDSolverCfg, +) diff --git a/source/isaaclab_contrib/isaaclab_contrib/deformable/coupled_featherstone_vbd_manager.py b/source/isaaclab_contrib/isaaclab_contrib/deformable/coupled_featherstone_vbd_manager.py new file mode 100644 index 00000000000..312d9fc69f1 --- /dev/null +++ b/source/isaaclab_contrib/isaaclab_contrib/deformable/coupled_featherstone_vbd_manager.py @@ -0,0 +1,509 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Coupled Featherstone + VBD Newton manager.""" + +from __future__ import annotations + +import inspect +import logging +from typing import TYPE_CHECKING + +import warp as wp +from isaaclab_newton.physics.newton_manager import NewtonManager +from newton import Contacts, Control, Model, ModelBuilder, State +from newton._src.usd.schemas import SchemaResolverNewton, SchemaResolverPhysx +from newton.solvers import SolverBase, SolverFeatherstone, SolverVBD + +from isaaclab.sim.utils.stage import get_current_stage + +from .deformable_object import ( + add_deformable_entry_to_builder, + clear_deformable_builder_hooks, + install_deformable_builder_hooks, +) +from .kernels import _kernel_body_particle_reaction +from .newton_manager_cfg import CoupledFeatherstoneVBDSolverCfg + +if TYPE_CHECKING: + from isaaclab.sim.simulation_context import SimulationContext + +logger = logging.getLogger(__name__) + + +class NewtonCoupledFeatherstoneVBDManager(NewtonManager): + """:class:`NewtonManager` specialization for the coupled Featherstone + VBD + solver. Due to Newton deformables not being properly integrated yet, this + manager uses the same temporary solutions from VBD Manager. + + Always uses Newton's :class:`CollisionPipeline` for contact handling. + """ + + _rigid_solver: SolverFeatherstone + _soft_solver: SolverVBD + _coupling_mode: str | None = None + + @classmethod + def initialize(cls, sim_context: SimulationContext) -> None: + """Initialize the manager with simulation context. + + Args: + sim_context: Parent simulation context. + + TODO: Subclass should not override this method, once deformables + supported on Newton import_usd, this can be unified with NewtonManager's + implementation. + """ + + # Deformable body registry and extension hooks. + # Experimental deformable support registers callbacks here so the manager + # and cloner can invoke them without hard-coding deformable logic. + install_deformable_builder_hooks() + + super().initialize(sim_context) + + @classmethod + def step(cls) -> None: + """Step the physics simulation.""" + from isaaclab.physics import PhysicsManager + + sim = PhysicsManager._sim + if sim is None or not sim.is_playing(): + return + + # Notify solver of model changes + if cls._model_changes: + with wp.ScopedDevice(PhysicsManager._device): + for change in cls._model_changes: + cls._rigid_solver.notify_model_changed(change) + cls._soft_solver.notify_model_changed(change) + NewtonManager._model_changes = set() + super().step() + + @classmethod + def _solver_specific_clear(cls): + """Clear VBD-specific state.""" + clear_deformable_builder_hooks() + + @classmethod + def _get_deformable_ignore_paths(cls) -> list[str]: + """Return USD prim paths to skip when calling ``builder.add_usd``. + + For each registered deformable body, both the simulation mesh (which + carries ``UsdPhysics.CollisionAPI``) and the visual mesh are returned. + The sim mesh must be skipped so Newton does not create a redundant + static mesh collider alongside the particles produced by + ``add_soft_mesh``. The visual mesh is skipped so Newton does not + treat it as a collider — Kit reads it directly from USD for rendering. + + Paths may contain regex patterns; Newton's ``add_usd`` matches them + via :func:`re.match`. + """ + paths: list[str] = [] + for entry in cls._deformable_registry: + paths.append(entry.sim_mesh_prim_path) + paths.append(entry.vis_mesh_prim_path) + return paths + + @classmethod + def start_simulation(cls) -> None: + """Start simulation by finalizing model and initializing state. + + This function finalizes the model and initializes the simulation state. + Note: Collision pipeline is initialized later in initialize_solver() after + we determine whether the solver needs external collision detection. + + TODO: Subclass should not override this method, missing piece is + having Newton bind a surface mesh to volume deformable tetrahedral mesh + in addition to removing the deformable_registry data structure. + """ + super().start_simulation() + + # Apply global model parameters from :class:`NewtonModelCfg` to the finalized model. + # Sets ``soft_contact_ke/kd/mu`` and optionally overrides per-shape + # ``shape_material_ke/kd/mu`` on the Newton model. + from isaaclab.physics import PhysicsManager + + cfg = PhysicsManager._cfg + if cfg is not None and hasattr(cfg, "model_cfg") and cfg.model_cfg is not None: + model = cls._model + if model is None: + return + + model_cfg = cfg.model_cfg + model.soft_contact_ke = float(model_cfg.soft_contact_ke) + model.soft_contact_kd = float(model_cfg.soft_contact_kd) + model.soft_contact_mu = float(model_cfg.soft_contact_mu) + + if model_cfg.shape_material_ke is not None: + model.shape_material_ke.fill_(float(model_cfg.shape_material_ke)) + if model_cfg.shape_material_kd is not None: + model.shape_material_kd.fill_(float(model_cfg.shape_material_kd)) + if model_cfg.shape_material_mu is not None: + model.shape_material_mu.fill_(float(model_cfg.shape_material_mu)) + + # Setup USD/Fabric sync for Kit viewport deformable rendering + if not cls._clone_physics_only and cls._deformable_registry: + import re + + import usdrt + + if NewtonManager._usdrt_stage is None: + NewtonManager._usdrt_stage = get_current_stage(fabric=True) + + stage = get_current_stage() + for entry in cls._deformable_registry: + for inst_idx, offset in enumerate(entry.particle_offsets): + # Resolve regex pattern to concrete instance path of visual mesh + resolved_vis = re.sub(r"(?<=[Ee]nv_)\.\*", str(inst_idx), entry.vis_mesh_prim_path) + resolved_vis = re.sub(r"\.\*", str(inst_idx), resolved_vis) + vis_prim = stage.GetPrimAtPath(resolved_vis) + + if not vis_prim or not vis_prim.IsValid(): + logger.warning("[setup_fabric_particle_sync] vis prim not found at %s", resolved_vis) + continue + + # Create per-instance particle offset and count attributes on the visual mesh + # prim so the Fabric sync kernel can find the right slice of particle_q + # and iterate only over this body's particles (counts vary across bodies). + fab_prim = NewtonManager._usdrt_stage.GetPrimAtPath(vis_prim.GetPath().pathString) + fab_prim.CreateAttribute( + NewtonManager._newton_particle_offset_attr, usdrt.Sdf.ValueTypeNames.UInt, True + ) + fab_prim.GetAttribute(NewtonManager._newton_particle_offset_attr).Set(offset) + fab_prim.CreateAttribute( + NewtonManager._newton_particle_count_attr, usdrt.Sdf.ValueTypeNames.UInt, True + ) + fab_prim.GetAttribute(NewtonManager._newton_particle_count_attr).Set(entry.particles_per_body) + + cls._mark_particles_dirty() + cls.sync_particles_to_usd() + + @classmethod + def instantiate_builder_from_stage(cls): + """Create builder from USD stage with special treatment for deformable + bodies, as these are not read from USD yet. + + Detects env Xforms (e.g. ``/World/Env_0``, ``/World/Env_1``) and builds + each as a separate Newton world via ``begin_world``/``end_world``. + Falls back to a flat ``add_usd`` when no env Xforms are found. + + TODO: Subclass should not override this method, once deformables + supported on Newton import_usd, this can be unified with NewtonManager's + implementation. + """ + import re + + from pxr import UsdGeom + + stage = get_current_stage() + up_axis = UsdGeom.GetStageUpAxis(stage) + + # Scan /World children for env-like Xforms (Env_0, env_1, ...) + env_pattern = re.compile(r"^[Ee]nv_(\d+)$") + world_prim = stage.GetPrimAtPath("/World") + env_paths: list[tuple[int, str]] = [] + if world_prim and world_prim.IsValid(): + for child in world_prim.GetChildren(): + m = env_pattern.match(child.GetName()) + if m: + env_paths.append((int(m.group(1)), child.GetPath().pathString)) + env_paths.sort(key=lambda x: x[0]) + + builder = ModelBuilder(up_axis=up_axis) + + schema_resolvers = [SchemaResolverNewton(), SchemaResolverPhysx()] + + # Deformable sim/visual mesh paths must be skipped by ``add_usd`` + # so they don't get duplicated as static colliders. + deformable_ignore_paths = cls._get_deformable_ignore_paths() + + if not env_paths: + # No env Xforms — flat loading + builder.add_usd(stage, ignore_paths=deformable_ignore_paths, schema_resolvers=schema_resolvers) + + # Add deformable bodies from the registry (single world at origin). + for entry in cls._deformable_registry: + add_deformable_entry_to_builder(builder, entry, 0, [0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 1.0]) + else: + # Load everything except the env subtrees (ground plane, lights, etc.) + ignore_paths = [path for _, path in env_paths] + deformable_ignore_paths + builder.add_usd(stage, ignore_paths=ignore_paths, schema_resolvers=schema_resolvers) + + # Build a prototype from the first env (all envs assumed identical) + _, proto_path = env_paths[0] + proto = ModelBuilder(up_axis=up_axis) + proto.add_usd( + stage, + root_path=proto_path, + ignore_paths=deformable_ignore_paths, + schema_resolvers=schema_resolvers, + ) + + # Inject registered sites into the proto before replication + global_sites, proto_sites = cls._cl_inject_sites(builder, {proto_path: proto}) + global_site_map: dict[str, tuple[int, None]] = {label: (idx, None) for label, idx in global_sites.items()} + num_worlds = len(env_paths) + local_site_map: dict[str, list[list[int]]] = {} + site_entries = proto_sites.get(id(proto), {}) + + # Add each env as a separate Newton world + xform_cache = UsdGeom.XformCache() + for col, (_, env_path) in enumerate(env_paths): + builder.begin_world() + offset = builder.shape_count + world_xform = xform_cache.GetLocalToWorldTransform(stage.GetPrimAtPath(env_path)) + translation = world_xform.ExtractTranslation() + rotation = world_xform.ExtractRotationQuat() + pos = (translation[0], translation[1], translation[2]) + quat = ( + rotation.GetImaginary()[0], + rotation.GetImaginary()[1], + rotation.GetImaginary()[2], + rotation.GetReal(), + ) + builder.add_builder(proto, xform=wp.transform(pos, quat)) + for label, proto_shape_indices in site_entries.items(): + if label not in local_site_map: + local_site_map[label] = [[] for _ in range(num_worlds)] + for proto_shape_idx in proto_shape_indices: + local_site_map[label][col].append(offset + proto_shape_idx) + + # Add deformable bodies from the registry into this world. + for entry in cls._deformable_registry: + add_deformable_entry_to_builder(builder, entry, col, list(pos), quat) + + builder.end_world() + + NewtonManager._cl_site_index_map = { + **global_site_map, + **{label: (None, per_world) for label, per_world in local_site_map.items()}, + } + NewtonManager._num_envs = len(env_paths) + + # Call builder.color() if any deformable entries were added (required by VBD solver) + if cls._deformable_registry: + builder.color() + + cls.set_builder(builder) + + @classmethod + def _build_solver(cls, model: Model, solver_cfg: CoupledFeatherstoneVBDSolverCfg) -> None: + """Construct a custom coupling between two solvers and populate the + base-class slots. + + VBD always uses Newton's :class:`CollisionPipeline` and steps with + separate input/output states, so the flags are fixed. + """ + cls._coupling_mode = solver_cfg.coupling_mode + + valid = set(inspect.signature(SolverFeatherstone.__init__).parameters) - {"self", "model"} + kwargs = {k: v for k, v in solver_cfg.rigid_solver_cfg.to_dict().items() if k in valid} + cls._rigid_solver = SolverFeatherstone(model, **kwargs) + + valid = set(inspect.signature(SolverVBD.__init__).parameters) - {"self", "model"} + kwargs = {k: v for k, v in solver_cfg.soft_solver_cfg.to_dict().items() if k in valid} + cls._soft_solver = SolverVBD(model, **kwargs) + + # Dummy solver for the newtonmanager + NewtonManager._solver = SolverBase(model) + + NewtonManager._use_single_state = False + NewtonManager._needs_collision_pipeline = True + + if solver_cfg.coupling_mode == "kinematic": + cls._gravity_zero = wp.zeros(1, dtype=wp.vec3) + cls._gravity_saved = wp.clone(model.gravity) + # Save original PD gains and create zeroed versions for kinematic step + cls._ke_saved = wp.clone(model.joint_target_ke) + cls._kd_saved = wp.clone(model.joint_target_kd) + cls._ke_zero = wp.zeros_like(model.joint_target_ke) + cls._kd_zero = wp.zeros_like(model.joint_target_kd) + + @classmethod + def _step_solver( + cls, state_in: State, state_out: State, control: Control, contacts: Contacts | None, substep_dt: float + ) -> None: + """One coupled substep. + + Args: + state_in: Current state (read/write). + state_out: Next state (write). + control: Joint-level control inputs. + contacts: Ignored -- the solver uses its own internal contacts. + dt: Substep timestep [s]. + """ + if cls._coupling_mode == "kinematic": + cls._step_kinematic(state_in, state_out, control, substep_dt) + elif cls._coupling_mode == "one_way": + cls._step_one_way(state_in, state_out, control, substep_dt) + else: + cls._step_two_way(state_in, state_out, control, substep_dt) + + @classmethod + def _simulate_physics_only(cls) -> None: + # Rebuild BVH once per step for solvers that require it (e.g. VBD cloth). + if hasattr(cls._soft_solver, "rebuild_bvh"): + cls._soft_solver.rebuild_bvh(cls._state_0) + super()._simulate_physics_only() + + @classmethod + def _step_kinematic(cls, state_in: State, state_out: State, control: Control, dt: float) -> None: + """Kinematic coupling: mirrors some Newton examples (e.g. softbody_franka) exactly. + + 1. Clear forces. + 2. Assign joint_qd from control targets (velocity = (target - current) / frame_dt). + 3. Disable gravity and rigid contacts for the rigid solver step. + 4. Step rigid solver as kinematic integrator (q += qd * dt). + 5. Restore gravity, collision detect, VBD step. + """ + model = cls._model + + # 1. Clear forces + state_in.clear_forces() + state_out.clear_forces() + + # 2. Kinematic rigid step: assign qd, disable gravity/contacts/PD gains + saved_particle_count = model.particle_count + saved_shape_contact_pair_count = model.shape_contact_pair_count + model.particle_count = 0 + model.gravity.assign(cls._gravity_zero) + model.shape_contact_pair_count = 0 + + # Zero out PD gains so rigid solver (Featherstone) acts as a pure kinematic integrator + model.joint_target_ke.assign(cls._ke_zero) + model.joint_target_kd.assign(cls._kd_zero) + + # Assign joint velocities from control targets + state_in.joint_qd.assign(control.joint_target_vel) + + cls._rigid_solver.step(state_in, state_out, control, None, dt) + + # 3. Restore everything + state_in.particle_f.zero_() + model.particle_count = saved_particle_count + model.gravity.assign(cls._gravity_saved) + model.shape_contact_pair_count = saved_shape_contact_pair_count + model.joint_target_ke.assign(cls._ke_saved) + model.joint_target_kd.assign(cls._kd_saved) + + # 4. Collision detection + cls._collision_pipeline.collide(state_in, cls._contacts) + + # 5. VBD step + cls._soft_solver.step(state_in, state_out, control, cls._contacts, dt) + + @classmethod + def _step_one_way(cls, state_in: State, state_out: State, control: Control, dt: float) -> None: + """One-way coupling: collide, then rigid step, then VBD.""" + # 1. Clear forces + state_in.clear_forces() + state_out.clear_forces() + + # 2. Collision detection (cloth-body contacts) + cls._collision_pipeline.collide(state_in, cls._contacts) + + # 3. Rigid-body step (does not read soft-contact reactions) + cls._rigid_step(state_in, state_out, control, dt) + + # 4. Clear spurious particle forces from rigid step + state_in.particle_f.zero_() + + # 5. VBD step -- particles only, reads updated rigid poses + cls._soft_solver.step(state_in, state_out, control, cls._contacts, dt) + + @classmethod + def _step_two_way(cls, state_in: State, state_out: State, control: Control, dt: float) -> None: + """Two-way coupling: collide, inject reactions into body_f, rigid step, VBD step.""" + # 1. Clear forces + state_in.clear_forces() + state_out.clear_forces() + + # 2. Collision detection BEFORE rigid step + cls._collision_pipeline.collide(state_in, cls._contacts) + + # 3. Inject contact reaction forces into body_f. + # state_out holds the previous substep's body_q (states swap each + # substep), used for finite-difference body velocity in friction. + # particle_q_prev is reconstructed from particle_qd inside the + # kernel because VBD mutates particle_q in place, so the swapped + # state's particle_q is not a clean prior-substep snapshot. + if state_in.body_f is not None: + cls._apply_reactions(state_in, state_out, dt) + + # 4. Rigid-body step (reads body_f for soft-contact reactions) + cls._rigid_step(state_in, state_out, control, dt) + + # 5. Clear spurious particle forces from rigid step + state_in.particle_f.zero_() + + # 6. VBD step -- uses same contacts detected in step 2 + cls._soft_solver.step(state_in, state_out, control, cls._contacts, dt) + + @classmethod + def _rigid_step(cls, state_in: State, state_out: State, control: Control, dt: float) -> None: + """Advance rigid bodies with the configured sub-solver.""" + model = cls._model + + # set particle_count = 0 to disable particle simulation in robot solver + saved_particle_count = model.particle_count + model.particle_count = 0 + + cls._rigid_solver.step(state_in, state_out, control, None, dt) + + # restore original settings + model.particle_count = saved_particle_count + + @classmethod + def _apply_reactions(cls, state: State, state_prev: State, dt: float) -> None: + """Launch the reaction kernel to inject normal + friction forces into body_f. + + Args: + state: Current state with particle positions/velocities and body state. + state_prev: Previous substep state whose ``body_q`` provides + the reference poses for finite-difference body velocity. + dt: Substep timestep [s]. + """ + model = cls._model + contacts = cls._contacts + + if contacts is None: + return + + contact_capacity = int(contacts.soft_contact_particle.shape[0]) + if contact_capacity == 0: + return + + # The kernel reconstructs particle_q_prev from particle_qd internally: + # state_prev.particle_q is unreliable because VBD mutates particle_q + # in place during its iteration, so the swapped state's particle_q is + # not a clean snapshot of the prior substep. + wp.launch( + _kernel_body_particle_reaction, + dim=contact_capacity, + inputs=[ + contacts.soft_contact_count, + contacts.soft_contact_particle, + contacts.soft_contact_shape, + contacts.soft_contact_body_pos, + contacts.soft_contact_body_vel, + contacts.soft_contact_normal, + state.particle_q, + state.particle_qd, + model.particle_radius, + state.body_q, + state_prev.body_q, + state.body_qd, + model.body_com, + model.shape_body, + model.shape_material_mu, + float(model.soft_contact_ke), + float(model.soft_contact_kd), + float(model.soft_contact_mu), + float(cls._soft_solver.friction_epsilon), + float(dt), + state.body_f, + ], + ) diff --git a/source/isaaclab_contrib/isaaclab_contrib/deformable/coupled_mjwarp_vbd_manager.py b/source/isaaclab_contrib/isaaclab_contrib/deformable/coupled_mjwarp_vbd_manager.py new file mode 100644 index 00000000000..5097284bafc --- /dev/null +++ b/source/isaaclab_contrib/isaaclab_contrib/deformable/coupled_mjwarp_vbd_manager.py @@ -0,0 +1,452 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Coupled MJWarp + VBD Newton manager.""" + +from __future__ import annotations + +import inspect +import logging +from typing import TYPE_CHECKING + +import warp as wp +from isaaclab_newton.physics.newton_manager import NewtonManager +from newton import Contacts, Control, Model, ModelBuilder, State +from newton._src.usd.schemas import SchemaResolverNewton, SchemaResolverPhysx +from newton.solvers import SolverBase, SolverMuJoCo, SolverVBD + +from isaaclab.sim.utils.stage import get_current_stage + +from .deformable_object import ( + add_deformable_entry_to_builder, + clear_deformable_builder_hooks, + install_deformable_builder_hooks, +) +from .kernels import _kernel_body_particle_reaction +from .newton_manager_cfg import CoupledMJWarpVBDSolverCfg + +if TYPE_CHECKING: + from isaaclab.sim.simulation_context import SimulationContext + +logger = logging.getLogger(__name__) + + +class NewtonCoupledMJWarpVBDManager(NewtonManager): + """:class:`NewtonManager` specialization for the coupled MJWarp + VBD + solver. Due to Newton deformables not being properly integrated yet, this + manager uses the same temporary solutions from VBD Manager. + + Always uses Newton's :class:`CollisionPipeline` for contact handling. + """ + + _rigid_solver: SolverMuJoCo + _soft_solver: SolverVBD + _coupling_mode: str | None = None + + @classmethod + def initialize(cls, sim_context: SimulationContext) -> None: + """Initialize the manager with simulation context. + + Args: + sim_context: Parent simulation context. + + TODO: Subclass should not override this method, once deformables + supported on Newton import_usd, this can be unified with NewtonManager's + implementation. + """ + + # Deformable body registry and extension hooks. + # Experimental deformable support registers callbacks here so the manager + # and cloner can invoke them without hard-coding deformable logic. + install_deformable_builder_hooks() + + super().initialize(sim_context) + + @classmethod + def step(cls) -> None: + """Step the physics simulation.""" + from isaaclab.physics import PhysicsManager + + sim = PhysicsManager._sim + if sim is None or not sim.is_playing(): + return + + # Notify solver of model changes + if cls._model_changes: + with wp.ScopedDevice(PhysicsManager._device): + for change in cls._model_changes: + cls._rigid_solver.notify_model_changed(change) + cls._soft_solver.notify_model_changed(change) + NewtonManager._model_changes = set() + super().step() + + @classmethod + def _solver_specific_clear(cls): + """Clear VBD-specific state.""" + clear_deformable_builder_hooks() + + @classmethod + def _get_deformable_ignore_paths(cls) -> list[str]: + """Return USD prim paths to skip when calling ``builder.add_usd``. + + For each registered deformable body, both the simulation mesh (which + carries ``UsdPhysics.CollisionAPI``) and the visual mesh are returned. + The sim mesh must be skipped so Newton does not create a redundant + static mesh collider alongside the particles produced by + ``add_soft_mesh``. The visual mesh is skipped so Newton does not + treat it as a collider — Kit reads it directly from USD for rendering. + + Paths may contain regex patterns; Newton's ``add_usd`` matches them + via :func:`re.match`. + """ + paths: list[str] = [] + for entry in cls._deformable_registry: + paths.append(entry.sim_mesh_prim_path) + paths.append(entry.vis_mesh_prim_path) + return paths + + @classmethod + def start_simulation(cls) -> None: + """Start simulation by finalizing model and initializing state. + + This function finalizes the model and initializes the simulation state. + Note: Collision pipeline is initialized later in initialize_solver() after + we determine whether the solver needs external collision detection. + + TODO: Subclass should not override this method, missing piece is + having Newton bind a surface mesh to volume deformable tetrahedral mesh + in addition to removing the deformable_registry data structure. + """ + super().start_simulation() + + # Apply global model parameters from :class:`NewtonModelCfg` to the finalized model. + # Sets ``soft_contact_ke/kd/mu`` and optionally overrides per-shape + # ``shape_material_ke/kd/mu`` on the Newton model. + from isaaclab.physics import PhysicsManager + + cfg = PhysicsManager._cfg + if cfg is not None and hasattr(cfg, "model_cfg") and cfg.model_cfg is not None: + model = cls._model + if model is None: + return + + model_cfg = cfg.model_cfg + model.soft_contact_ke = float(model_cfg.soft_contact_ke) + model.soft_contact_kd = float(model_cfg.soft_contact_kd) + model.soft_contact_mu = float(model_cfg.soft_contact_mu) + + if model_cfg.shape_material_ke is not None: + model.shape_material_ke.fill_(float(model_cfg.shape_material_ke)) + if model_cfg.shape_material_kd is not None: + model.shape_material_kd.fill_(float(model_cfg.shape_material_kd)) + if model_cfg.shape_material_mu is not None: + model.shape_material_mu.fill_(float(model_cfg.shape_material_mu)) + + # Setup USD/Fabric sync for Kit viewport deformable rendering + if not cls._clone_physics_only and cls._deformable_registry: + import re + + import usdrt + + if NewtonManager._usdrt_stage is None: + NewtonManager._usdrt_stage = get_current_stage(fabric=True) + + stage = get_current_stage() + for entry in cls._deformable_registry: + for inst_idx, offset in enumerate(entry.particle_offsets): + # Resolve regex pattern to concrete instance path of visual mesh + resolved_vis = re.sub(r"(?<=[Ee]nv_)\.\*", str(inst_idx), entry.vis_mesh_prim_path) + resolved_vis = re.sub(r"\.\*", str(inst_idx), resolved_vis) + vis_prim = stage.GetPrimAtPath(resolved_vis) + + if not vis_prim or not vis_prim.IsValid(): + logger.warning("[setup_fabric_particle_sync] vis prim not found at %s", resolved_vis) + continue + + # Create per-instance particle offset and count attributes on the visual mesh + # prim so the Fabric sync kernel can find the right slice of particle_q + # and iterate only over this body's particles (counts vary across bodies). + fab_prim = NewtonManager._usdrt_stage.GetPrimAtPath(vis_prim.GetPath().pathString) + fab_prim.CreateAttribute( + NewtonManager._newton_particle_offset_attr, usdrt.Sdf.ValueTypeNames.UInt, True + ) + fab_prim.GetAttribute(NewtonManager._newton_particle_offset_attr).Set(offset) + fab_prim.CreateAttribute( + NewtonManager._newton_particle_count_attr, usdrt.Sdf.ValueTypeNames.UInt, True + ) + fab_prim.GetAttribute(NewtonManager._newton_particle_count_attr).Set(entry.particles_per_body) + + cls._mark_particles_dirty() + cls.sync_particles_to_usd() + + @classmethod + def instantiate_builder_from_stage(cls): + """Create builder from USD stage with special treatment for deformable + bodies, as these are not read from USD yet. + + Detects env Xforms (e.g. ``/World/Env_0``, ``/World/Env_1``) and builds + each as a separate Newton world via ``begin_world``/``end_world``. + Falls back to a flat ``add_usd`` when no env Xforms are found. + + TODO: Subclass should not override this method, once deformables + supported on Newton import_usd, this can be unified with NewtonManager's + implementation. + """ + import re + + from pxr import UsdGeom + + stage = get_current_stage() + up_axis = UsdGeom.GetStageUpAxis(stage) + + # Scan /World children for env-like Xforms (Env_0, env_1, ...) + env_pattern = re.compile(r"^[Ee]nv_(\d+)$") + world_prim = stage.GetPrimAtPath("/World") + env_paths: list[tuple[int, str]] = [] + if world_prim and world_prim.IsValid(): + for child in world_prim.GetChildren(): + m = env_pattern.match(child.GetName()) + if m: + env_paths.append((int(m.group(1)), child.GetPath().pathString)) + env_paths.sort(key=lambda x: x[0]) + + builder = ModelBuilder(up_axis=up_axis) + + schema_resolvers = [SchemaResolverNewton(), SchemaResolverPhysx()] + + # Deformable sim/visual mesh paths must be skipped by ``add_usd`` + # so they don't get duplicated as static colliders. + deformable_ignore_paths = cls._get_deformable_ignore_paths() + + if not env_paths: + # No env Xforms — flat loading + builder.add_usd(stage, ignore_paths=deformable_ignore_paths, schema_resolvers=schema_resolvers) + + # Add deformable bodies from the registry (single world at origin). + for entry in cls._deformable_registry: + add_deformable_entry_to_builder(builder, entry, 0, [0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 1.0]) + else: + # Load everything except the env subtrees (ground plane, lights, etc.) + ignore_paths = [path for _, path in env_paths] + deformable_ignore_paths + builder.add_usd(stage, ignore_paths=ignore_paths, schema_resolvers=schema_resolvers) + + # Build a prototype from the first env (all envs assumed identical) + _, proto_path = env_paths[0] + proto = ModelBuilder(up_axis=up_axis) + proto.add_usd( + stage, + root_path=proto_path, + ignore_paths=deformable_ignore_paths, + schema_resolvers=schema_resolvers, + ) + + # Inject registered sites into the proto before replication + global_sites, proto_sites = cls._cl_inject_sites(builder, {proto_path: proto}) + global_site_map: dict[str, tuple[int, None]] = {label: (idx, None) for label, idx in global_sites.items()} + num_worlds = len(env_paths) + local_site_map: dict[str, list[list[int]]] = {} + site_entries = proto_sites.get(id(proto), {}) + + # Add each env as a separate Newton world + xform_cache = UsdGeom.XformCache() + for col, (_, env_path) in enumerate(env_paths): + builder.begin_world() + offset = builder.shape_count + world_xform = xform_cache.GetLocalToWorldTransform(stage.GetPrimAtPath(env_path)) + translation = world_xform.ExtractTranslation() + rotation = world_xform.ExtractRotationQuat() + pos = (translation[0], translation[1], translation[2]) + quat = ( + rotation.GetImaginary()[0], + rotation.GetImaginary()[1], + rotation.GetImaginary()[2], + rotation.GetReal(), + ) + builder.add_builder(proto, xform=wp.transform(pos, quat)) + for label, proto_shape_indices in site_entries.items(): + if label not in local_site_map: + local_site_map[label] = [[] for _ in range(num_worlds)] + for proto_shape_idx in proto_shape_indices: + local_site_map[label][col].append(offset + proto_shape_idx) + + # Add deformable bodies from the registry into this world. + for entry in cls._deformable_registry: + add_deformable_entry_to_builder(builder, entry, col, list(pos), quat) + + builder.end_world() + + NewtonManager._cl_site_index_map = { + **global_site_map, + **{label: (None, per_world) for label, per_world in local_site_map.items()}, + } + NewtonManager._num_envs = len(env_paths) + + # Call builder.color() if any deformable entries were added (required by VBD solver) + if cls._deformable_registry: + builder.color() + + cls.set_builder(builder) + + @classmethod + def _build_solver(cls, model: Model, solver_cfg: CoupledMJWarpVBDSolverCfg) -> None: + """Construct a custom coupling between two solvers and populate the + base-class slots. + + VBD always uses Newton's :class:`CollisionPipeline` and steps with + separate input/output states, so the flags are fixed. + """ + cls._coupling_mode = solver_cfg.coupling_mode + + valid = set(inspect.signature(SolverMuJoCo.__init__).parameters) - {"self", "model"} + kwargs = {k: v for k, v in solver_cfg.rigid_solver_cfg.to_dict().items() if k in valid} + cls._rigid_solver = SolverMuJoCo(model, **kwargs) + + valid = set(inspect.signature(SolverVBD.__init__).parameters) - {"self", "model"} + kwargs = {k: v for k, v in solver_cfg.soft_solver_cfg.to_dict().items() if k in valid} + cls._soft_solver = SolverVBD(model, **kwargs) + + # Dummy solver for the newtonmanager + NewtonManager._solver = SolverBase(model) + + NewtonManager._use_single_state = False + NewtonManager._needs_collision_pipeline = True + + @classmethod + def _step_solver( + cls, state_in: State, state_out: State, control: Control, contacts: Contacts | None, substep_dt: float + ) -> None: + """One coupled substep. + + Args: + state_in: Current state (read/write). + state_out: Next state (write). + control: Joint-level control inputs. + contacts: Ignored -- the solver uses its own internal contacts. + dt: Substep timestep [s]. + """ + if cls._coupling_mode == "one_way": + cls._step_one_way(state_in, state_out, control, substep_dt) + else: + cls._step_two_way(state_in, state_out, control, substep_dt) + + @classmethod + def _simulate_physics_only(cls) -> None: + # Rebuild BVH once per step for solvers that require it (e.g. VBD cloth). + if hasattr(cls._soft_solver, "rebuild_bvh"): + cls._soft_solver.rebuild_bvh(cls._state_0) + super()._simulate_physics_only() + + @classmethod + def _step_one_way(cls, state_in: State, state_out: State, control: Control, dt: float) -> None: + """One-way coupling: collide, then rigid step, then VBD.""" + # 1. Clear forces + state_in.clear_forces() + state_out.clear_forces() + + # 2. Collision detection (cloth-body contacts) + cls._collision_pipeline.collide(state_in, cls._contacts) + + # 3. Rigid-body step (does not read soft-contact reactions) + cls._rigid_step(state_in, state_out, control, dt) + + # 4. Clear spurious particle forces from rigid step + state_in.particle_f.zero_() + + # 5. VBD step -- particles only, reads updated rigid poses + cls._soft_solver.step(state_in, state_out, control, cls._contacts, dt) + + @classmethod + def _step_two_way(cls, state_in: State, state_out: State, control: Control, dt: float) -> None: + """Two-way coupling: collide, inject reactions into body_f, rigid step, VBD step.""" + # 1. Clear forces + state_in.clear_forces() + state_out.clear_forces() + + # 2. Collision detection BEFORE rigid step + cls._collision_pipeline.collide(state_in, cls._contacts) + + # 3. Inject contact reaction forces into body_f. + # state_out holds the previous substep's body_q (states swap each + # substep), used for finite-difference body velocity in friction. + # particle_q_prev is reconstructed from particle_qd inside the + # kernel because VBD mutates particle_q in place, so the swapped + # state's particle_q is not a clean prior-substep snapshot. + if state_in.body_f is not None: + cls._apply_reactions(state_in, state_out, dt) + + # 4. Rigid-body step (reads body_f for soft-contact reactions) + cls._rigid_step(state_in, state_out, control, dt) + + # 5. Clear spurious particle forces from rigid step + state_in.particle_f.zero_() + + # 6. VBD step -- uses same contacts detected in step 2 + cls._soft_solver.step(state_in, state_out, control, cls._contacts, dt) + + @classmethod + def _rigid_step(cls, state_in: State, state_out: State, control: Control, dt: float) -> None: + """Advance rigid bodies with the configured sub-solver.""" + model = cls._model + + # set particle_count = 0 to disable particle simulation in robot solver + saved_particle_count = model.particle_count + model.particle_count = 0 + + cls._rigid_solver.step(state_in, state_out, control, None, dt) + + # restore original settings + model.particle_count = saved_particle_count + + @classmethod + def _apply_reactions(cls, state: State, state_prev: State, dt: float) -> None: + """Launch the reaction kernel to inject normal + friction forces into body_f. + + Args: + state: Current state with particle positions/velocities and body state. + state_prev: Previous substep state whose ``body_q`` provides + the reference poses for finite-difference body velocity. + dt: Substep timestep [s]. + """ + model = cls._model + contacts = cls._contacts + + if contacts is None: + return + + contact_capacity = int(contacts.soft_contact_particle.shape[0]) + if contact_capacity == 0: + return + + # The kernel reconstructs particle_q_prev from particle_qd internally: + # state_prev.particle_q is unreliable because VBD mutates particle_q + # in place during its iteration, so the swapped state's particle_q is + # not a clean snapshot of the prior substep. + wp.launch( + _kernel_body_particle_reaction, + dim=contact_capacity, + inputs=[ + contacts.soft_contact_count, + contacts.soft_contact_particle, + contacts.soft_contact_shape, + contacts.soft_contact_body_pos, + contacts.soft_contact_body_vel, + contacts.soft_contact_normal, + state.particle_q, + state.particle_qd, + model.particle_radius, + state.body_q, + state_prev.body_q, + state.body_qd, + model.body_com, + model.shape_body, + model.shape_material_mu, + float(model.soft_contact_ke), + float(model.soft_contact_kd), + float(model.soft_contact_mu), + float(cls._soft_solver.friction_epsilon), + float(dt), + state.body_f, + ], + ) diff --git a/source/isaaclab_contrib/isaaclab_contrib/deformable/deformable_object.py b/source/isaaclab_contrib/isaaclab_contrib/deformable/deformable_object.py new file mode 100644 index 00000000000..3c7fd10eb03 --- /dev/null +++ b/source/isaaclab_contrib/isaaclab_contrib/deformable/deformable_object.py @@ -0,0 +1,962 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +from __future__ import annotations + +import logging +from collections.abc import Sequence +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +import numpy as np +import torch +import warp as wp +from isaaclab_newton.physics import NewtonManager as SimulationManager + +import isaaclab.sim as sim_utils +from isaaclab.assets.deformable_object.base_deformable_object import BaseDeformableObject +from isaaclab.markers import VisualizationMarkers +from isaaclab.physics import PhysicsEvent +from isaaclab.utils.warp import ProxyArray + +from .deformable_object_data import DeformableObjectData +from .kernels import ( + compute_nodal_state_w, + enforce_kinematic_targets, + scatter_particles_state_vec6f_mask, + scatter_particles_vec3f_index, + scatter_particles_vec3f_mask, + set_kinematic_flags_to_one, + vec6f, + write_nodal_kinematic_target_index, + write_nodal_kinematic_target_mask, +) + + +@dataclass +class DeformableRegistryEntry: + """Entry in the deformable body registry. + + Registered by :class:`DeformableObject` during ``__init__``, consumed by + ``newton_physics_replicate`` inside the per-world ``begin_world``/``end_world`` loop. + After replication, ``particle_offsets`` and ``particles_per_body`` are filled in + so the asset can bind to the correct particle ranges. + """ + + prim_path: str + sim_mesh_prim_path: str + vis_mesh_prim_path: str + vertices: list + indices: list + init_pos: tuple[float, float, float] + init_rot: tuple[float, float, float, float] # (x, y, z, w) + deformable_type: str | None = None # "volume" or "surface" + # Cloth params + density: float = 1.0 + tri_ke: float = 1e4 + tri_ka: float = 1e4 + tri_kd: float = 1.5e-6 + edge_ke: float = 5.0 + edge_kd: float = 1e-2 + particle_radius: float = 0.008 + # Tet params + k_mu: float = 1e5 + k_lambda: float = 1e5 + k_damp: float = 0.0 + # Filled by newton_physics_replicate: + particle_offsets: list[int] = field(default_factory=list) + particles_per_body: int = 0 + + +if TYPE_CHECKING: + from isaaclab.assets.deformable_object.deformable_object_cfg import DeformableObjectCfg + +logger = logging.getLogger(__name__) + + +def add_deformable_entry_to_builder( + builder, + entry: DeformableRegistryEntry, + env_idx: int, + env_position: list[float], + env_rotation: list[float] | tuple[float, float, float, float], +) -> None: + """Add a deformable registry entry to a Newton ``ModelBuilder`` for one environment. + + Depending on the deformable type (``"volume"`` or ``"surface"``), calls + ``builder.add_soft_mesh()`` or ``builder.add_cloth_mesh()`` with the mesh + data and material properties stored in the registry entry. + + Also records the particle offset for the instance and, on the first + environment, records the per-body particle count. + + Args: + builder: The Newton ``ModelBuilder``. + entry: A :class:`DeformableRegistryEntry` with mesh data and config. + env_idx: The environment index. + env_position: World position [x, y, z] [m] for this environment. + env_rotation: World orientation as quaternion ``(x, y, z, w)`` for this environment. + """ + if env_idx == 0: + entry.particle_offsets.clear() + entry.particles_per_body = 0 + + before_count = getattr(builder, "particle_count", 0) + + env_pos = wp.vec3(float(env_position[0]), float(env_position[1]), float(env_position[2])) + env_rot = wp.quat( + float(env_rotation[0]), + float(env_rotation[1]), + float(env_rotation[2]), + float(env_rotation[3]), + ) + init_pos = wp.vec3(float(entry.init_pos[0]), float(entry.init_pos[1]), float(entry.init_pos[2])) + init_rot = wp.quat( + float(entry.init_rot[0]), + float(entry.init_rot[1]), + float(entry.init_rot[2]), + float(entry.init_rot[3]), + ) + body_pos = env_pos + wp.quat_rotate(env_rot, init_pos) + body_rot = env_rot * init_rot + + if entry.deformable_type == "volume": + builder.add_soft_mesh( + pos=body_pos, + rot=body_rot, + scale=1.0, + vel=wp.vec3(0.0, 0.0, 0.0), + vertices=entry.vertices, + indices=entry.indices, + density=entry.density, + k_mu=entry.k_mu, + k_lambda=entry.k_lambda, + k_damp=entry.k_damp, + particle_radius=entry.particle_radius, + ) + elif entry.deformable_type == "surface": + builder.add_cloth_mesh( + pos=body_pos, + rot=body_rot, + scale=1.0, + vel=wp.vec3(0.0, 0.0, 0.0), + vertices=entry.vertices, + indices=entry.indices, + density=entry.density, + tri_ke=entry.tri_ke, + tri_ka=entry.tri_ka, + tri_kd=entry.tri_kd, + edge_ke=entry.edge_ke, + edge_kd=entry.edge_kd, + particle_radius=entry.particle_radius, + ) + else: + raise ValueError( + f"Invalid deformable type '{entry.deformable_type}' for registry entry with prim path '{entry.prim_path}'" + ) + + after_count = getattr(builder, "particle_count", 0) + delta = after_count - before_count + + entry.particle_offsets.append(before_count) + if env_idx == 0: + entry.particles_per_body = delta + elif entry.particles_per_body != delta: + raise RuntimeError( + f"Deformable body '{entry.prim_path}' produced {delta} particles in env {env_idx}, " + f"but env 0 produced {entry.particles_per_body}." + ) + + +def add_registered_deformables_to_builder( + builder, + world_idx: int, + env_position: list[float], + env_rotation: list[float] | tuple[float, float, float, float], +) -> None: + """Add all registered deformable entries to one Newton builder world.""" + for entry in SimulationManager._deformable_registry: + add_deformable_entry_to_builder(builder, entry, world_idx, env_position, env_rotation) + + +def color_registered_deformables(builder) -> None: + """Color the Newton builder when deformables were registered.""" + if SimulationManager._deformable_registry: + builder.color() + + +def install_deformable_builder_hooks() -> None: + """Install deformable builder hooks without removing hooks owned by other extensions.""" + SimulationManager._deformable_registry = [] + if not hasattr(SimulationManager, "_per_world_builder_hooks"): + SimulationManager._per_world_builder_hooks = [] + if not hasattr(SimulationManager, "_post_replicate_hooks"): + SimulationManager._post_replicate_hooks = [] + if add_registered_deformables_to_builder not in SimulationManager._per_world_builder_hooks: + SimulationManager._per_world_builder_hooks.append(add_registered_deformables_to_builder) + if color_registered_deformables not in SimulationManager._post_replicate_hooks: + SimulationManager._post_replicate_hooks.append(color_registered_deformables) + + +def clear_deformable_builder_hooks() -> None: + """Clear deformable registry state and remove only deformable-owned builder hooks.""" + SimulationManager._deformable_registry = [] + if hasattr(SimulationManager, "_per_world_builder_hooks"): + SimulationManager._per_world_builder_hooks = [ + hook + for hook in SimulationManager._per_world_builder_hooks + if hook is not add_registered_deformables_to_builder + ] + if hasattr(SimulationManager, "_post_replicate_hooks"): + SimulationManager._post_replicate_hooks = [ + hook for hook in SimulationManager._post_replicate_hooks if hook is not color_registered_deformables + ] + + +class DeformableObject(BaseDeformableObject): + """A deformable object asset class (Newton backend). + + This class manages cloth/deformable bodies in the Newton physics engine. Newton stores all + particles in flat arrays (``state.particle_q``, ``state.particle_qd``). This class builds + a per-instance indexing layer on top of those flat arrays, enabling the standard + :class:`BaseDeformableObject` interface for reading/writing nodal state. + + The cloth mesh is added to the Newton :class:`ModelBuilder` during the ``MODEL_INIT`` phase. + The mesh data is read from the USD prim at :attr:`cfg.prim_path`, and cloth simulation + parameters (density, stiffness, etc.) come from :attr:`DeformableObjectCfg`. + """ + + cfg: DeformableObjectCfg + """Configuration instance for the deformable object.""" + + __backend_name__: str = "newton" + """The name of the backend for the deformable object.""" + + def __init__(self, cfg: DeformableObjectCfg): + """Initialize the deformable object. + + Args: + cfg: A configuration instance. + """ + super().__init__(cfg) + + # initialize deformable type to None, should be set to either surface or volume on initialization + self._deformable_type: str | None = None + + # Read mesh from the spawned USD prim and register in the deformable registry. + self._registry_entry = self._register_deformable() + + # Register custom vec6f type for nodal state validation. + self._DTYPE_TO_TORCH_TRAILING_DIMS = {**self._DTYPE_TO_TORCH_TRAILING_DIMS, vec6f: (6,)} + + """ + Properties + """ + + @property + def data(self) -> DeformableObjectData: + return self._data + + @property + def num_instances(self) -> int: + return self._num_instances + + @property + def num_bodies(self) -> int: + """Number of bodies in the asset. + + This is always 1 since each object is a single deformable body. + """ + return 1 + + @property + def max_sim_vertices_per_body(self) -> int: + """The maximum number of simulation mesh vertices per deformable body.""" + return self._particles_per_body + + """ + Operations. + """ + + def reset(self, env_ids: Sequence[int] | None = None, env_mask: wp.array | None = None) -> None: + """Reset the deformable object. + + No-op to match the PhysX deformable object convention. + + Args: + env_ids: Environment indices. If None, then all indices are used. + env_mask: Environment mask. If None, then all the instances are updated. + Shape is (num_instances,). + """ + pass + + def write_data_to_sim(self): + """Apply kinematic targets to the Newton simulation. + + Reads the stored kinematic target buffer and enforces it on particles: + kinematic particles (flag=0) get inv_mass=0, particle_flags=0, target position, + and zero velocity; free particles (flag=1) get their original inv_mass and + particle_flags=1 (ACTIVE) restored. + + Writes to both ``state_0`` and ``state_1`` so kinematic positions survive + the state swaps that happen between substeps. + """ + if ( + self._data.nodal_kinematic_target is None + or self._default_particle_inv_mass is None + or self._default_particle_flags is None + ): + return + + model = SimulationManager.get_model() + if model is None: + return + + for state in self._iter_particle_states(): + wp.launch( + enforce_kinematic_targets, + dim=(self._num_instances, self._particles_per_body), + inputs=[ + self._data.nodal_kinematic_target.warp, + self._particle_offsets, + self._default_particle_inv_mass, + self._default_particle_flags, + ], + outputs=[ + state.particle_q, + state.particle_qd, + model.particle_inv_mass, + model.particle_flags, + ], + device=self.device, + ) + + def update(self, dt: float): + self._data.update(dt) + + """ + Operations - Write to simulation. + """ + + def write_nodal_pos_to_sim_index( + self, + nodal_pos: torch.Tensor | wp.array | ProxyArray, + env_ids: Sequence[int] | torch.Tensor | wp.array | None = None, + full_data: bool = False, + ) -> None: + """Set the nodal positions over selected environment indices into the simulation. + + Args: + nodal_pos: Nodal positions in simulation frame [m]. + Shape is (len(env_ids), max_sim_vertices_per_body, 3) + or (num_instances, max_sim_vertices_per_body, 3). + env_ids: Environment indices. If None, then all indices are used. + full_data: Whether to expect full data. Defaults to False. + """ + env_ids = self._resolve_env_ids(env_ids) + if isinstance(nodal_pos, ProxyArray): + nodal_pos = nodal_pos.warp + if full_data: + self.assert_shape_and_dtype( + nodal_pos, (self.num_instances, self._particles_per_body), wp.vec3f, "nodal_pos" + ) + else: + self.assert_shape_and_dtype(nodal_pos, (env_ids.shape[0], self._particles_per_body), wp.vec3f, "nodal_pos") + if isinstance(nodal_pos, torch.Tensor): + nodal_pos = wp.from_torch(nodal_pos.contiguous(), dtype=wp.vec3f) + + for state in self._iter_particle_states(): + wp.launch( + scatter_particles_vec3f_index, + dim=(env_ids.shape[0], self._particles_per_body), + inputs=[nodal_pos, env_ids, self._particle_offsets, full_data], + outputs=[state.particle_q], + device=self.device, + ) + + self._invalidate_nodal_pos_cache() + + def write_nodal_velocity_to_sim_index( + self, + nodal_vel: torch.Tensor | wp.array | ProxyArray, + env_ids: Sequence[int] | torch.Tensor | wp.array | None = None, + full_data: bool = False, + ) -> None: + """Set the nodal velocity over selected environment indices into the simulation. + + Args: + nodal_vel: Nodal velocities in simulation frame [m/s]. + Shape is (len(env_ids), max_sim_vertices_per_body, 3) + or (num_instances, max_sim_vertices_per_body, 3). + env_ids: Environment indices. If None, then all indices are used. + full_data: Whether to expect full data. Defaults to False. + """ + env_ids = self._resolve_env_ids(env_ids) + if isinstance(nodal_vel, ProxyArray): + nodal_vel = nodal_vel.warp + if full_data: + self.assert_shape_and_dtype( + nodal_vel, (self.num_instances, self._particles_per_body), wp.vec3f, "nodal_vel" + ) + else: + self.assert_shape_and_dtype(nodal_vel, (env_ids.shape[0], self._particles_per_body), wp.vec3f, "nodal_vel") + if isinstance(nodal_vel, torch.Tensor): + nodal_vel = wp.from_torch(nodal_vel.contiguous(), dtype=wp.vec3f) + + for state in self._iter_particle_states(): + wp.launch( + scatter_particles_vec3f_index, + dim=(env_ids.shape[0], self._particles_per_body), + inputs=[nodal_vel, env_ids, self._particle_offsets, full_data], + outputs=[state.particle_qd], + device=self.device, + ) + + self._invalidate_nodal_vel_cache() + + def write_nodal_kinematic_target_to_sim_index( + self, + targets: torch.Tensor | wp.array | ProxyArray, + env_ids: Sequence[int] | torch.Tensor | wp.array | None = None, + full_data: bool = False, + ) -> None: + """Set the kinematic targets of the simulation mesh for the deformable bodies. + + Newton has no native kinematic target API. Instead: + - Kinematic (flag=0.0): set ``particle_inv_mass`` to 0, write target pos, zero vel + - Free (flag=1.0): restore original ``particle_inv_mass`` + + Args: + targets: The kinematic targets comprising of nodal positions and flags [m]. + Shape is (len(env_ids), max_sim_vertices_per_body, 4) + or (num_instances, max_sim_vertices_per_body, 4). + env_ids: Environment indices. If None, then all indices are used. + full_data: Whether to expect full data. Defaults to False. + """ + env_ids = self._resolve_env_ids(env_ids) + if isinstance(targets, ProxyArray): + targets = targets.warp + if full_data: + self.assert_shape_and_dtype(targets, (self.num_instances, self._particles_per_body), wp.vec4f, "targets") + else: + self.assert_shape_and_dtype(targets, (env_ids.shape[0], self._particles_per_body), wp.vec4f, "targets") + if isinstance(targets, torch.Tensor): + if targets.dim() == 2: + targets = targets.unsqueeze(0) + targets = wp.from_torch(targets.contiguous(), dtype=wp.vec4f) + + # Store kinematic targets in our data buffer + if self._data.nodal_kinematic_target is not None: + wp.launch( + write_nodal_kinematic_target_index, + dim=(env_ids.shape[0], self._particles_per_body), + inputs=[targets, env_ids, full_data], + outputs=[self._data.nodal_kinematic_target.warp], + device=self.device, + ) + + """ + Operations - Write to simulation (mask variants). + """ + + def write_nodal_state_to_sim_mask( + self, + nodal_state: torch.Tensor | wp.array | ProxyArray, + env_mask: wp.array | torch.Tensor | None = None, + ) -> None: + """Set the nodal state over selected environment mask into the simulation. + + Args: + nodal_state: Nodal state in simulation frame [m, m/s]. + Shape is (num_instances, max_sim_vertices_per_body, 6). + env_mask: Environment mask. If None, then all indices are used. + Shape is (num_instances,). + """ + env_mask = self._resolve_mask(env_mask, self._ALL_ENV_MASK) + if isinstance(nodal_state, ProxyArray): + nodal_state = nodal_state.warp + self.assert_shape_and_dtype(nodal_state, (env_mask.shape[0], self._particles_per_body), vec6f, "nodal_state") + if isinstance(nodal_state, torch.Tensor): + nodal_state = wp.from_torch(nodal_state.contiguous(), dtype=vec6f) + + for state in self._iter_particle_states(): + wp.launch( + scatter_particles_state_vec6f_mask, + dim=(env_mask.shape[0], self._particles_per_body), + inputs=[nodal_state, env_mask, self._particle_offsets], + outputs=[state.particle_q, state.particle_qd], + device=self.device, + ) + + self._invalidate_nodal_state_cache() + + def write_nodal_pos_to_sim_mask( + self, + nodal_pos: torch.Tensor | wp.array | ProxyArray, + env_mask: wp.array | torch.Tensor | None = None, + ) -> None: + """Set the nodal positions over selected environment mask into the simulation. + + Args: + nodal_pos: Nodal positions in simulation frame [m]. + Shape is (num_instances, max_sim_vertices_per_body, 3). + env_mask: Environment mask. If None, then all indices are used. + Shape is (num_instances,). + """ + env_mask = self._resolve_mask(env_mask, self._ALL_ENV_MASK) + if isinstance(nodal_pos, ProxyArray): + nodal_pos = nodal_pos.warp + self.assert_shape_and_dtype(nodal_pos, (env_mask.shape[0], self._particles_per_body), wp.vec3f, "nodal_pos") + if isinstance(nodal_pos, torch.Tensor): + nodal_pos = wp.from_torch(nodal_pos.contiguous(), dtype=wp.vec3f) + + for state in self._iter_particle_states(): + wp.launch( + scatter_particles_vec3f_mask, + dim=(env_mask.shape[0], self._particles_per_body), + inputs=[nodal_pos, env_mask, self._particle_offsets], + outputs=[state.particle_q], + device=self.device, + ) + + self._invalidate_nodal_pos_cache() + + def write_nodal_velocity_to_sim_mask( + self, + nodal_vel: torch.Tensor | wp.array | ProxyArray, + env_mask: wp.array | torch.Tensor | None = None, + ) -> None: + """Set the nodal velocity over selected environment mask into the simulation. + + Args: + nodal_vel: Nodal velocities in simulation frame [m/s]. + Shape is (num_instances, max_sim_vertices_per_body, 3). + env_mask: Environment mask. If None, then all indices are used. + Shape is (num_instances,). + """ + env_mask = self._resolve_mask(env_mask, self._ALL_ENV_MASK) + if isinstance(nodal_vel, ProxyArray): + nodal_vel = nodal_vel.warp + self.assert_shape_and_dtype(nodal_vel, (env_mask.shape[0], self._particles_per_body), wp.vec3f, "nodal_vel") + if isinstance(nodal_vel, torch.Tensor): + nodal_vel = wp.from_torch(nodal_vel.contiguous(), dtype=wp.vec3f) + + for state in self._iter_particle_states(): + wp.launch( + scatter_particles_vec3f_mask, + dim=(env_mask.shape[0], self._particles_per_body), + inputs=[nodal_vel, env_mask, self._particle_offsets], + outputs=[state.particle_qd], + device=self.device, + ) + + self._invalidate_nodal_vel_cache() + + def write_nodal_kinematic_target_to_sim_mask( + self, + targets: torch.Tensor | wp.array | ProxyArray, + env_mask: wp.array | torch.Tensor | None = None, + ) -> None: + """Set the kinematic targets over selected environment mask into the target buffer. + + Args: + targets: The kinematic targets comprising of nodal positions and flags [m]. + Shape is (num_instances, max_sim_vertices_per_body, 4). + env_mask: Environment mask. If None, then all indices are used. + Shape is (num_instances,). + """ + env_mask = self._resolve_mask(env_mask, self._ALL_ENV_MASK) + if isinstance(targets, ProxyArray): + targets = targets.warp + self.assert_shape_and_dtype(targets, (env_mask.shape[0], self._particles_per_body), wp.vec4f, "targets") + if isinstance(targets, torch.Tensor): + targets = wp.from_torch(targets.contiguous(), dtype=wp.vec4f) + + if self._data.nodal_kinematic_target is not None: + wp.launch( + write_nodal_kinematic_target_mask, + dim=(env_mask.shape[0], self._particles_per_body), + inputs=[targets, env_mask], + outputs=[self._data.nodal_kinematic_target.warp], + device=self.device, + ) + + """ + Internal helper. + """ + + def _resolve_env_ids(self, env_ids): + """Resolve environment indices to a warp int32 array.""" + if env_ids is None or (isinstance(env_ids, slice) and env_ids == slice(None)): + return self._ALL_INDICES + elif isinstance(env_ids, list): + return wp.array(env_ids, dtype=wp.int32, device=self.device) + elif isinstance(env_ids, torch.Tensor): + return wp.from_torch(env_ids.to(torch.int32), dtype=wp.int32) + return env_ids + + def _resolve_mask(self, mask: wp.array | torch.Tensor | None, full_mask: wp.array) -> wp.array: + """Resolve an environment mask to a warp bool array.""" + if mask is None: + return full_mask + if isinstance(mask, torch.Tensor): + if mask.dtype != torch.bool: + mask = mask.to(torch.bool) + return wp.from_torch(mask, dtype=wp.bool) + return mask + + def _iter_particle_states(self): + """Yield active Newton states.""" + for state in (SimulationManager.get_state_0(), SimulationManager.get_state_1()): + if state is None: + continue + yield state + + def _invalidate_nodal_pos_cache(self) -> None: + """Invalidate cached position-derived deformable data.""" + self._data._nodal_pos_w.timestamp = -1.0 + self._data._nodal_state_w.timestamp = -1.0 + self._data._root_pos_w.timestamp = -1.0 + + def _invalidate_nodal_vel_cache(self) -> None: + """Invalidate cached velocity-derived deformable data.""" + self._data._nodal_vel_w.timestamp = -1.0 + self._data._nodal_state_w.timestamp = -1.0 + self._data._root_vel_w.timestamp = -1.0 + + def _invalidate_nodal_state_cache(self) -> None: + """Invalidate all cached nodal state data.""" + self._invalidate_nodal_pos_cache() + self._invalidate_nodal_vel_cache() + + def _register_deformable(self) -> DeformableRegistryEntry: + """Read mesh from the spawned USD prim and register in NewtonManager's deformable registry. + + Returns: + The registry entry (also stored on NewtonManager._deformable_registry). + + Note: + pxr imports are deferred to this method (not module level) so that + ``resolve_task_config`` can import the env-cfg module before Kit + starts without polluting the ``pxr`` module cache. + """ + from pxr import Gf, UsdGeom, UsdShade + + # Resolve the path of the actually-spawned template prim. This must mirror + # :meth:`AssetBase.__init__`: ``spawn_path`` is set by ``InteractiveScene`` + # when the asset is part of the template-based cloning flow (the spawn + # lives at ``/World/template//proto_asset_*`` and per-env clones at + # ``/World/envs/env_*/`` are not yet authored). For Direct envs + # that spawn straight at the cloned regex, ``spawn_path`` is unset, so + # we fall back to ``prim_path`` — which already matches the spawned prim. + # The cloned-regex ``cfg.prim_path`` is still used below to build the + # registry entry's :attr:`sim_mesh_prim_path` / :attr:`vis_mesh_prim_path` + # so post-replicate consumers resolve all per-env clones. + lookup_path = ( + self.cfg.spawn.spawn_path + if self.cfg.spawn is not None and self.cfg.spawn.spawn_path is not None + else self.cfg.prim_path + ) + template_prim = sim_utils.find_first_matching_prim(lookup_path) + if template_prim is None: + raise RuntimeError(f"Failed to find prim for expression: '{lookup_path}'.") + template_prim_path = template_prim.GetPrimPath() + + # Discover sim / visual mesh prims under the template. + # The spawner authors a visual UsdGeom.Mesh and a separate simulation mesh + # (UsdGeom.TetMesh for volume, UsdGeom.Mesh for surface) with a + # ``*DeformableSimAPI`` applied, so we split candidates by that schema. + def _is_sim_mesh(prim) -> bool: + return any("DeformableSimAPI" in api for api in prim.GetAppliedSchemas()) + + tet_prims = sim_utils.get_all_matching_child_prims(template_prim_path, lambda p: p.GetTypeName() == "TetMesh") + mesh_prims = sim_utils.get_all_matching_child_prims(template_prim_path, lambda p: p.GetTypeName() == "Mesh") + + if len(tet_prims) > 1: + raise ValueError( + f"Found multiple TetMesh prims under '{template_prim_path}': " + f"{[p.GetPrimPath() for p in tet_prims]}." + " Deformable body schema supports only one simulation mesh per asset." + ) + + # Pick simulation and visual mesh prims. + if len(tet_prims) == 1: + deformable_type = "volume" + mesh_prim = tet_prims[0] + vis_candidates = [p for p in mesh_prims if not _is_sim_mesh(p)] + elif len(mesh_prims) > 0: + deformable_type = "surface" + sim_candidates = [p for p in mesh_prims if _is_sim_mesh(p)] + vis_candidates = [p for p in mesh_prims if not _is_sim_mesh(p)] + if len(sim_candidates) > 1: + raise ValueError( + f"Found multiple simulation Mesh prims under '{template_prim_path}': " + f"{[p.GetPrimPath() for p in sim_candidates]}." + " Deformable body schema supports only one simulation mesh per asset." + ) + # Fall back to the single authored Mesh when no explicit sim mesh was tagged + # (legacy / self-simulated surfaces where the visual mesh *is* the sim mesh). + mesh_prim = sim_candidates[0] if sim_candidates else vis_candidates[0] + if not sim_candidates: + vis_candidates = [] # visual == sim, no separate embedding target + else: + raise ValueError( + f"Could not find any surface or volume mesh in '{template_prim_path}'. Please check asset." + ) + + # Revert visual and simulation mesh prim paths back to template-relative form for registry storage, + # since the actual prim paths will differ per world instance after replication. + # When vis_candidates is empty the visual mesh IS the simulation mesh + # (e.g. a plain surface cloth with no separate visual embedding). + vis_mesh_prim = vis_candidates[0] if vis_candidates else mesh_prim + vis_mesh_prim_path = str(vis_mesh_prim.GetPrimPath()) + vis_mesh_prim_path = self.cfg.prim_path + vis_mesh_prim_path[len(template_prim_path.pathString) :] + sim_mesh_prim_path = str(mesh_prim.GetPrimPath()) + sim_mesh_prim_path = self.cfg.prim_path + sim_mesh_prim_path[len(template_prim_path.pathString) :] + logger.info("Registered visual UsdGeom.Mesh at %s.", vis_mesh_prim_path) + + # Bake the template prim's xform directly into the vertex positions. + xform_cache = UsdGeom.XformCache() + mesh_to_parent_frame = ( + xform_cache.GetLocalToWorldTransform(mesh_prim) + * xform_cache.GetLocalToWorldTransform(template_prim.GetParent()).GetInverse() + ) + + def _bake_points(raw_pts) -> list[wp.vec3]: + out = [] + for p in raw_pts: + q = mesh_to_parent_frame.Transform(Gf.Vec3d(float(p[0]), float(p[1]), float(p[2]))) + out.append(wp.vec3(float(q[0]), float(q[1]), float(q[2]))) + return out + + if deformable_type == "volume": + tet_mesh = UsdGeom.TetMesh(mesh_prim) + pts = tet_mesh.GetPointsAttr().Get() + vertices = _bake_points(pts) + raw_tet_indices = tet_mesh.GetTetVertexIndicesAttr().Get() + indices = [] + for vec4i in raw_tet_indices: + indices.extend([int(vec4i[0]), int(vec4i[1]), int(vec4i[2]), int(vec4i[3])]) + logger.info("Registered UsdGeom.TetMesh: %d vertices, %d tetrahedra.", len(pts), len(indices) // 4) + else: # surface + usd_mesh = UsdGeom.Mesh(mesh_prim) + pts = usd_mesh.GetPointsAttr().Get() + vertices = _bake_points(pts) + indices = list(usd_mesh.GetFaceVertexIndicesAttr().Get()) + logger.info("Registered UsdGeom.Mesh: %d vertices.", len(pts)) + + # init_pos/init_rot are already baked into the vertices by the Xform + # transform above. Setting them to identity prevents add_cloth_mesh/add_soft_mesh + # from applying them a second time. + # Note: add_deformable_entry_to_builder passes init_rot directly to + # wp.quat(x, y, z, w), so identity must be (0, 0, 0, 1) not (1, 0, 0, 0). + init_pos = (0.0, 0.0, 0.0) + init_rot = (0.0, 0.0, 0.0, 1.0) + + # Look up the bound deformable physics material + if not template_prim.HasAPI(UsdShade.MaterialBindingAPI): + raise ValueError( + f"Template prim '{template_prim_path}' must have a UsdShade.MaterialBindingAPI applied" + " with a Newton deformable physics material target." + ) + material_targets = UsdShade.MaterialBindingAPI(template_prim).GetDirectBindingRel("physics").GetTargets() + stage = template_prim.GetStage() + material_prim = None + for mat_path in material_targets: + mat_prim = stage.GetPrimAtPath(mat_path) + if mat_prim.GetAttribute("newton:density").IsValid(): + material_prim = mat_prim + break + if material_prim is None: + raise ValueError( + f"Could not find a Newton deformable physics material" + f" among the physics material targets of '{template_prim_path}'." + ) + + def _get_material_attr(name: str, default): + attr = material_prim.GetAttribute(name) + return attr.Get() if attr.IsValid() else default + + density = _get_material_attr("newton:density", DeformableRegistryEntry.density) + particle_radius = _get_material_attr("newton:particleRadius", DeformableRegistryEntry.particle_radius) + k_mu = _get_material_attr("newton:kMu", DeformableRegistryEntry.k_mu) + k_lambda = _get_material_attr("newton:kLambda", DeformableRegistryEntry.k_lambda) + k_damp = _get_material_attr("newton:kDamp", DeformableRegistryEntry.k_damp) + + tri_ke = _get_material_attr("newton:triKe", DeformableRegistryEntry.tri_ke) + tri_ka = _get_material_attr("newton:triKa", DeformableRegistryEntry.tri_ka) + tri_kd = _get_material_attr("newton:triKd", DeformableRegistryEntry.tri_kd) + edge_ke = _get_material_attr("newton:edgeKe", DeformableRegistryEntry.edge_ke) + edge_kd = _get_material_attr("newton:edgeKd", DeformableRegistryEntry.edge_kd) + + entry = DeformableRegistryEntry( + prim_path=self.cfg.prim_path, + sim_mesh_prim_path=sim_mesh_prim_path, + vis_mesh_prim_path=vis_mesh_prim_path, + vertices=vertices, + indices=indices, + deformable_type=deformable_type, + init_pos=init_pos, + init_rot=init_rot, + density=density, + tri_ke=tri_ke, + tri_ka=tri_ka, + tri_kd=tri_kd, + edge_ke=edge_ke, + edge_kd=edge_kd, + particle_radius=particle_radius, + k_mu=k_mu, + k_lambda=k_lambda, + k_damp=k_damp, + ) + SimulationManager._deformable_registry.append(entry) + self._deformable_type = deformable_type + return entry + + def _initialize_impl(self): + """Initialize physics handles and buffers after the Newton model is ready.""" + entry = self._registry_entry + self._num_instances = len(entry.particle_offsets) + self._particles_per_body = entry.particles_per_body + self._recorded_particle_offsets = entry.particle_offsets + + if self._num_instances == 0: + raise RuntimeError( + f"No deformable body instances found for '{self.cfg.prim_path}'. " + "Ensure newton_physics_replicate or MODEL_INIT processed the registry." + ) + + logger.info("Newton deformable object initialized at: %s", self.cfg.prim_path) + logger.info("Number of instances: %d", self._num_instances) + logger.info("Particles per body: %d", self._particles_per_body) + + # Build particle offset array on device + self._particle_offsets = wp.array(self._recorded_particle_offsets, dtype=wp.int32, device=self.device) + + # Create data container + self._data = DeformableObjectData( + particle_offsets=self._particle_offsets, + particles_per_body=self._particles_per_body, + num_instances=self._num_instances, + device=self.device, + ) + + # Create buffers + self._create_buffers() + + # Update data once + self.update(0.0) + + # Register rebind callback for full resets + self._physics_ready_handle = SimulationManager.register_callback( + lambda _: self._data._create_simulation_bindings(), + PhysicsEvent.PHYSICS_READY, + name=f"deformable_object_rebind_{self.cfg.prim_path}", + ) + + def _create_buffers(self): + """Create buffers for storing data.""" + # Constants + self._ALL_INDICES = wp.array(np.arange(self._num_instances, dtype=np.int32), device=self.device) + self._ALL_ENV_MASK = wp.ones((self._num_instances,), dtype=wp.bool, device=self.device) + + # Snapshot default positions from current state (after finalize + FK) + state = SimulationManager.get_state_0() + if state is not None and state.particle_q is not None: + from .kernels import gather_particles_vec3f + + self._default_nodal_pos_w = wp.zeros( + (self._num_instances, self._particles_per_body), dtype=wp.vec3f, device=self.device + ) + wp.launch( + gather_particles_vec3f, + dim=(self._num_instances, self._particles_per_body), + inputs=[state.particle_q, self._particle_offsets, self._particles_per_body], + outputs=[self._default_nodal_pos_w], + device=self.device, + ) + + # Compute default nodal state as vec6f (positions + zero velocities) + nodal_velocities = wp.zeros( + (self._num_instances, self._particles_per_body), dtype=wp.vec3f, device=self.device + ) + default_nodal_state_w = wp.zeros( + (self._num_instances, self._particles_per_body), dtype=vec6f, device=self.device + ) + wp.launch( + compute_nodal_state_w, + dim=(self._num_instances, self._particles_per_body), + inputs=[self._default_nodal_pos_w, nodal_velocities], + outputs=[default_nodal_state_w], + device=self.device, + ) + self._data.default_nodal_state_w = ProxyArray(default_nodal_state_w) + else: + self._default_nodal_pos_w = None + + # Snapshot default particle_inv_mass for kinematic target restoration + model = SimulationManager.get_model() + if model is not None and hasattr(model, "particle_inv_mass") and model.particle_inv_mass is not None: + self._default_particle_inv_mass = wp.clone(model.particle_inv_mass) + else: + self._default_particle_inv_mass = None + if model is not None and hasattr(model, "particle_flags") and model.particle_flags is not None: + self._default_particle_flags = wp.clone(model.particle_flags) + else: + self._default_particle_flags = None + + # Kinematic targets -- allocate and initialize with free flags + nodal_kinematic_target = wp.zeros( + (self._num_instances, self._particles_per_body), dtype=wp.vec4f, device=self.device + ) + wp.launch( + set_kinematic_flags_to_one, + dim=(self._num_instances * self._particles_per_body,), + inputs=[nodal_kinematic_target.reshape((self._num_instances * self._particles_per_body,))], + device=self.device, + ) + self._data.nodal_kinematic_target = ProxyArray(nodal_kinematic_target) + + # Set up the model parameters + model = SimulationManager.get_model() + if model is not None: + if hasattr(model, "edge_rest_angle"): + model.edge_rest_angle.zero_() + + """ + Internal simulation callbacks. + """ + + def _set_debug_vis_impl(self, debug_vis: bool): + if debug_vis: + if not hasattr(self, "target_visualizer"): + self.target_visualizer = VisualizationMarkers(self.cfg.visualizer_cfg) + self.target_visualizer.set_visibility(True) + else: + if hasattr(self, "target_visualizer"): + self.target_visualizer.set_visibility(False) + + def _debug_vis_callback(self, event): + num_enabled = 0 + if self._deformable_type == "volume": + kinematic_target_torch = self.data.nodal_kinematic_target.torch + targets_enabled = kinematic_target_torch[:, :, 3] == 0.0 + num_enabled = int(torch.sum(targets_enabled).item()) + if num_enabled == 0: + positions = torch.tensor([[0.0, 0.0, -10.0]], device=self.device) + else: + positions = kinematic_target_torch[targets_enabled][..., :3] + self.target_visualizer.visualize(positions) + + def _clear_callbacks(self) -> None: + """Clears all registered callbacks.""" + super()._clear_callbacks() + if hasattr(self, "_physics_ready_handle") and self._physics_ready_handle is not None: + self._physics_ready_handle.deregister() + self._physics_ready_handle = None + + def _invalidate_initialize_callback(self, event): + """Invalidates the scene elements.""" + super()._invalidate_initialize_callback(event) diff --git a/source/isaaclab_contrib/isaaclab_contrib/deformable/deformable_object_data.py b/source/isaaclab_contrib/isaaclab_contrib/deformable/deformable_object_data.py new file mode 100644 index 00000000000..d516f1fc76d --- /dev/null +++ b/source/isaaclab_contrib/isaaclab_contrib/deformable/deformable_object_data.py @@ -0,0 +1,206 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +from __future__ import annotations + +import warp as wp +from isaaclab_newton.physics import NewtonManager as SimulationManager + +from isaaclab.assets.deformable_object.base_deformable_object_data import BaseDeformableObjectData +from isaaclab.utils.buffers import TimestampedBufferWarp as TimestampedBuffer +from isaaclab.utils.warp import ProxyArray + +from .kernels import compute_mean_vec3f_over_vertices, compute_nodal_state_w, gather_particles_vec3f, vec6f + + +class DeformableObjectData(BaseDeformableObjectData): + """Data container for a deformable object (Newton backend). + + Newton stores all particles in flat arrays (``model.particle_q``, ``state.particle_qd``). + This data class builds a per-instance view by gathering from the flat arrays using + precomputed offsets. + + The data is lazily updated, meaning that the data is only updated when it is accessed. + """ + + def __init__( + self, + particle_offsets: wp.array, + particles_per_body: int, + num_instances: int, + device: str, + ): + """Initialize the Newton deformable object data. + + Args: + particle_offsets: Per-instance start offset into the flat particle array. + Shape is (num_instances,) with dtype int32. + particles_per_body: Number of particles per deformable body instance. + num_instances: Number of deformable body instances. + device: The device used for processing. + """ + super().__init__(device) + + # Store dimensions and indexing + self._particle_offsets = particle_offsets + self._particles_per_body = particles_per_body + self._num_instances = num_instances + + # Initialize lazy buffers + self._nodal_pos_w = TimestampedBuffer((num_instances, particles_per_body), device, wp.vec3f) + self._nodal_vel_w = TimestampedBuffer((num_instances, particles_per_body), device, wp.vec3f) + self._nodal_state_w = TimestampedBuffer((num_instances, particles_per_body), device, vec6f) + self._root_pos_w = TimestampedBuffer((num_instances,), device, wp.vec3f) + self._root_vel_w = TimestampedBuffer((num_instances,), device, wp.vec3f) + self._nodal_pos_w_ta: ProxyArray | None = None + self._nodal_vel_w_ta: ProxyArray | None = None + self._nodal_state_w_ta: ProxyArray | None = None + self._root_pos_w_ta: ProxyArray | None = None + self._root_vel_w_ta: ProxyArray | None = None + + self._create_simulation_bindings() + + ## + # Defaults. + ## + + default_nodal_state_w: ProxyArray = None + """Default nodal state ``[nodal_pos, nodal_vel]`` in simulation world frame. + Shape is (num_instances, particles_per_body) with dtype vec6f. + """ + + ## + # Kinematic commands. + ## + + nodal_kinematic_target: ProxyArray = None + """Simulation mesh kinematic targets for the deformable bodies. + Shape is (num_instances, particles_per_body) with dtype vec4f. + """ + + def _create_simulation_bindings(self) -> None: + """Validate the current Newton particle state and invalidate gathered buffers. + + Newton may swap :attr:`state_0` and :attr:`state_1` across substeps, so deformable data does not keep + long-lived particle array bindings. Read properties query :meth:`SimulationManager.get_state_0` at gather time + and materialize object-local views from the current flat particle arrays. + """ + self._get_current_particle_state() + + # Invalidate lazy buffers gathered from the previous simulation state. + self._nodal_pos_w.timestamp = -1.0 + self._nodal_vel_w.timestamp = -1.0 + self._nodal_state_w.timestamp = -1.0 + self._root_pos_w.timestamp = -1.0 + self._root_vel_w.timestamp = -1.0 + + def _get_current_particle_state(self): + """Return the current Newton state containing deformable particle arrays.""" + state = SimulationManager.get_state_0() + if state is None or state.particle_q is None or state.particle_qd is None: + raise RuntimeError( + "Failed to access Newton deformable particle state. Ensure the Newton model has been finalized and " + "contains particle position and velocity arrays." + ) + return state + + ## + # Properties. + ## + + @property + def nodal_pos_w(self) -> ProxyArray: + """Nodal positions in simulation world frame [m]. Shape is (num_instances, particles_per_body) vec3f.""" + if self._nodal_pos_w.timestamp < self._sim_timestamp: + state = self._get_current_particle_state() + wp.launch( + gather_particles_vec3f, + dim=(self._num_instances, self._particles_per_body), + inputs=[state.particle_q, self._particle_offsets, self._particles_per_body], + outputs=[self._nodal_pos_w.data], + device=self.device, + ) + self._nodal_pos_w.timestamp = self._sim_timestamp + if self._nodal_pos_w_ta is None: + self._nodal_pos_w_ta = ProxyArray(self._nodal_pos_w.data) + return self._nodal_pos_w_ta + + @property + def nodal_vel_w(self) -> ProxyArray: + """Nodal velocities in simulation world frame [m/s]. Shape is (num_instances, particles_per_body) vec3f.""" + if self._nodal_vel_w.timestamp < self._sim_timestamp: + state = self._get_current_particle_state() + wp.launch( + gather_particles_vec3f, + dim=(self._num_instances, self._particles_per_body), + inputs=[state.particle_qd, self._particle_offsets, self._particles_per_body], + outputs=[self._nodal_vel_w.data], + device=self.device, + ) + self._nodal_vel_w.timestamp = self._sim_timestamp + if self._nodal_vel_w_ta is None: + self._nodal_vel_w_ta = ProxyArray(self._nodal_vel_w.data) + return self._nodal_vel_w_ta + + @property + def nodal_state_w(self) -> ProxyArray: + """Nodal state ``[nodal_pos, nodal_vel]`` in simulation world frame [m, m/s]. + + Shape is (num_instances, particles_per_body) vec6f. + """ + if self._nodal_state_w.timestamp < self._sim_timestamp: + wp.launch( + compute_nodal_state_w, + dim=(self._num_instances, self._particles_per_body), + inputs=[self.nodal_pos_w.warp, self.nodal_vel_w.warp], + outputs=[self._nodal_state_w.data], + device=self.device, + ) + self._nodal_state_w.timestamp = self._sim_timestamp + if self._nodal_state_w_ta is None: + self._nodal_state_w_ta = ProxyArray(self._nodal_state_w.data) + return self._nodal_state_w_ta + + ## + # Derived properties. + ## + + @property + def root_pos_w(self) -> ProxyArray: + """Root position from nodal positions [m]. Shape is (num_instances,) vec3f. + + This quantity is computed as the mean of the nodal positions. + """ + if self._root_pos_w.timestamp < self._sim_timestamp: + wp.launch( + compute_mean_vec3f_over_vertices, + dim=(self._num_instances,), + inputs=[self.nodal_pos_w.warp, self._particles_per_body], + outputs=[self._root_pos_w.data], + device=self.device, + ) + self._root_pos_w.timestamp = self._sim_timestamp + if self._root_pos_w_ta is None: + self._root_pos_w_ta = ProxyArray(self._root_pos_w.data) + return self._root_pos_w_ta + + @property + def root_vel_w(self) -> ProxyArray: + """Root velocity from nodal velocities [m/s]. Shape is (num_instances,) vec3f. + + This quantity is computed as the mean of the nodal velocities. + """ + if self._root_vel_w.timestamp < self._sim_timestamp: + wp.launch( + compute_mean_vec3f_over_vertices, + dim=(self._num_instances,), + inputs=[self.nodal_vel_w.warp, self._particles_per_body], + outputs=[self._root_vel_w.data], + device=self.device, + ) + self._root_vel_w.timestamp = self._sim_timestamp + if self._root_vel_w_ta is None: + self._root_vel_w_ta = ProxyArray(self._root_vel_w.data) + return self._root_vel_w_ta diff --git a/source/isaaclab_contrib/isaaclab_contrib/deformable/kernels.py b/source/isaaclab_contrib/isaaclab_contrib/deformable/kernels.py new file mode 100644 index 00000000000..179f62e5336 --- /dev/null +++ b/source/isaaclab_contrib/isaaclab_contrib/deformable/kernels.py @@ -0,0 +1,384 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Warp kernels for Newton deformable object gather/scatter operations.""" + +import warp as wp +from newton._src.solvers.vbd.rigid_vbd_kernels import ( + evaluate_body_particle_contact as _evaluate_body_particle_contact, +) + +vec6f = wp.types.vector(length=6, dtype=wp.float32) + + +@wp.kernel +def gather_particles_vec3f( + src: wp.array(dtype=wp.vec3f), + offsets: wp.array(dtype=wp.int32), + num_particles: int, + dst: wp.array2d(dtype=wp.vec3f), +): + """Gather particle data from a flat array into a per-instance 2D array. + + Args: + src: Flat source particle array (all instances concatenated). Shape is (total_particles,). + offsets: Per-instance start offset into the flat array. Shape is (num_instances,). + num_particles: Number of particles per instance. + dst: Output 2D array. Shape is (num_instances, num_particles). + """ + i, j = wp.tid() + dst[i, j] = src[offsets[i] + j] + + +@wp.kernel +def scatter_particles_vec3f_index( + src: wp.array2d(dtype=wp.vec3f), + env_ids: wp.array(dtype=wp.int32), + offsets: wp.array(dtype=wp.int32), + full_data: bool, + dst: wp.array(dtype=wp.vec3f), +): + """Scatter per-instance particle data into the flat simulation array using indices. + + Args: + src: Input 2D array. Shape is (len(env_ids), num_particles) or (num_instances, num_particles). + env_ids: Environment indices to scatter to. Shape is (num_selected,). + offsets: Per-instance start offset into the flat array. Shape is (num_instances,). + full_data: If True, index src with env_ids[i]; otherwise index src with i. + dst: Flat destination particle array. Shape is (total_particles,). + """ + i, j = wp.tid() + env_id = env_ids[i] + if full_data: + dst[offsets[env_id] + j] = src[env_id, j] + else: + dst[offsets[env_id] + j] = src[i, j] + + +@wp.kernel +def scatter_particles_vec3f_mask( + src: wp.array2d(dtype=wp.vec3f), + env_mask: wp.array(dtype=wp.bool), + offsets: wp.array(dtype=wp.int32), + dst: wp.array(dtype=wp.vec3f), +): + """Scatter per-instance particle data into the flat simulation array using a mask. + + Args: + src: Input particle data [m] or [m/s]. Shape is (num_instances, num_particles). + env_mask: Environment mask. Shape is (num_instances,). + offsets: Per-instance start offset into the flat array. Shape is (num_instances,). + dst: Flat destination particle array. Shape is (total_particles,). + """ + i, j = wp.tid() + if env_mask[i]: + dst[offsets[i] + j] = src[i, j] + + +@wp.kernel +def scatter_particles_state_vec6f_mask( + src: wp.array2d(dtype=vec6f), + env_mask: wp.array(dtype=wp.bool), + offsets: wp.array(dtype=wp.int32), + particle_q: wp.array(dtype=wp.vec3f), + particle_qd: wp.array(dtype=wp.vec3f), +): + """Scatter per-instance nodal state into the flat simulation arrays using a mask. + + Args: + src: Input nodal state data [m, m/s]. Shape is (num_instances, num_particles). + env_mask: Environment mask. Shape is (num_instances,). + offsets: Per-instance start offset into the flat arrays. Shape is (num_instances,). + particle_q: Flat destination particle positions [m]. Shape is (total_particles,). + particle_qd: Flat destination particle velocities [m/s]. Shape is (total_particles,). + """ + i, j = wp.tid() + if env_mask[i]: + state = src[i, j] + flat_idx = offsets[i] + j + particle_q[flat_idx] = wp.vec3f(state[0], state[1], state[2]) + particle_qd[flat_idx] = wp.vec3f(state[3], state[4], state[5]) + + +@wp.kernel +def write_nodal_kinematic_target_mask( + src: wp.array2d(dtype=wp.vec4f), + env_mask: wp.array(dtype=wp.bool), + dst: wp.array2d(dtype=wp.vec4f), +): + """Write kinematic target data into the per-instance target buffer using a mask. + + Args: + src: Input kinematic targets [m]. Shape is (num_instances, num_particles). + env_mask: Environment mask. Shape is (num_instances,). + dst: Destination kinematic target buffer [m]. Shape is (num_instances, num_particles). + """ + i, j = wp.tid() + if env_mask[i]: + dst[i, j] = src[i, j] + + +@wp.kernel +def write_nodal_kinematic_target_index( + src: wp.array2d(dtype=wp.vec4f), + env_ids: wp.array(dtype=wp.int32), + full_data: bool, + dst: wp.array2d(dtype=wp.vec4f), +): + """Write kinematic target data into the per-instance target buffer using indices. + + Args: + src: Input kinematic targets [m]. Shape is (len(env_ids), num_particles) + or (num_instances, num_particles). + env_ids: Environment indices to write. Shape is (num_selected,). + full_data: If True, index src with env_ids[i]; otherwise index src with i. + dst: Destination kinematic target buffer [m]. Shape is (num_instances, num_particles). + """ + i, j = wp.tid() + env_id = env_ids[i] + if full_data: + dst[env_id, j] = src[env_id, j] + else: + dst[env_id, j] = src[i, j] + + +@wp.kernel +def compute_nodal_state_w( + nodal_pos: wp.array2d(dtype=wp.vec3f), + nodal_vel: wp.array2d(dtype=wp.vec3f), + nodal_state: wp.array2d(dtype=vec6f), +): + """Concatenate nodal positions and velocities into a 6-element state vector. + + Args: + nodal_pos: Input array of nodal positions. Shape is (num_instances, num_vertices). + nodal_vel: Input array of nodal velocities. Shape is (num_instances, num_vertices). + nodal_state: Output array where concatenated state vectors are written. + Shape is (num_instances, num_vertices). + """ + i, j = wp.tid() + p = nodal_pos[i, j] + v = nodal_vel[i, j] + nodal_state[i, j] = vec6f(p[0], p[1], p[2], v[0], v[1], v[2]) + + +@wp.kernel +def compute_mean_vec3f_over_vertices( + data: wp.array2d(dtype=wp.vec3f), + num_vertices: int, + result: wp.array(dtype=wp.vec3f), +): + """Compute the mean of vec3f data over the vertex dimension. + + Args: + data: Input array of vec3f data. Shape is (num_instances, num_vertices). + num_vertices: Number of vertices per instance. + result: Output array where mean values are written. Shape is (num_instances,). + """ + i = wp.tid() + acc = wp.vec3f(0.0, 0.0, 0.0) + for j in range(num_vertices): + acc = acc + data[i, j] + result[i] = acc / float(num_vertices) + + +@wp.kernel +def scatter_zero_vel_index( + env_ids: wp.array(dtype=wp.int32), + offsets: wp.array(dtype=wp.int32), + num_particles: int, + dst: wp.array(dtype=wp.vec3f), +): + """Zero the velocity of particles for selected environments. + + Args: + env_ids: Environment indices to zero velocities for. Shape is (num_selected,). + offsets: Per-instance start offset into the flat array. Shape is (num_instances,). + num_particles: Number of particles per instance. + dst: Flat destination velocity array. Shape is (total_particles,). + """ + i, j = wp.tid() + env_id = env_ids[i] + dst[offsets[env_id] + j] = wp.vec3f(0.0, 0.0, 0.0) + + +@wp.kernel +def scatter_default_pos_index( + default_pos: wp.array2d(dtype=wp.vec3f), + env_ids: wp.array(dtype=wp.int32), + offsets: wp.array(dtype=wp.int32), + dst: wp.array(dtype=wp.vec3f), +): + """Scatter default positions for selected environments into the flat simulation array. + + Args: + default_pos: Default positions per instance. Shape is (num_instances, num_particles). + env_ids: Environment indices to reset. Shape is (num_selected,). + offsets: Per-instance start offset into the flat array. Shape is (num_instances,). + dst: Flat destination particle array. Shape is (total_particles,). + """ + i, j = wp.tid() + env_id = env_ids[i] + dst[offsets[env_id] + j] = default_pos[env_id, j] + + +@wp.kernel +def set_kinematic_flags_to_one( + data: wp.array(dtype=wp.vec4f), +): + """Set the w-component (kinematic flag) of all vec4f entries to 1.0. + + This is used to initialize all vertices as non-kinematic (free) nodes. + + Args: + data: Input/output array of vec4f kinematic targets. Shape is (N*V,). + """ + i = wp.tid() + v = data[i] + data[i] = wp.vec4f(v[0], v[1], v[2], 1.0) + + +@wp.kernel +def enforce_kinematic_targets( + targets: wp.array2d(dtype=wp.vec4f), + offsets: wp.array(dtype=wp.int32), + default_inv_mass: wp.array(dtype=wp.float32), + default_flags: wp.array(dtype=wp.int32), + particle_q: wp.array(dtype=wp.vec3f), + particle_qd: wp.array(dtype=wp.vec3f), + particle_inv_mass: wp.array(dtype=wp.float32), + particle_flags: wp.array(dtype=wp.int32), +): + """Enforce kinematic targets on Newton particles. + + For each particle, reads the kinematic target flag (w-component): + - flag == 0.0 (kinematic): set inv_mass to 0, particle_flags to 0, write target position, zero velocity. + - flag != 0.0 (free): restore the default inv_mass and set particle_flags to 1 (ACTIVE). + + Args: + targets: Per-instance kinematic targets. Shape is (num_instances, particles_per_body). + Each vec4f contains (target_x, target_y, target_z, flag). + offsets: Per-instance start offset into the flat particle array. + default_inv_mass: Saved default inverse masses. Shape is (total_particles,). + default_flags: Saved default particle flags. Shape is (total_particles,). + particle_q: Flat particle positions to write. Shape is (total_particles,). + particle_qd: Flat particle velocities to write. Shape is (total_particles,). + particle_inv_mass: Flat particle inverse masses to write. Shape is (total_particles,). + particle_flags: Flat particle flags to write. Shape is (total_particles,). + 0 = kinematic (solver skips integration), 1 = ACTIVE. + """ + i, j = wp.tid() + t = targets[i, j] + flat_idx = offsets[i] + j + flag = t[3] + if flag == 0.0: + particle_inv_mass[flat_idx] = 0.0 + particle_flags[flat_idx] = 0 + particle_q[flat_idx] = wp.vec3f(t[0], t[1], t[2]) + particle_qd[flat_idx] = wp.vec3f(0.0, 0.0, 0.0) + else: + particle_inv_mass[flat_idx] = default_inv_mass[flat_idx] + particle_flags[flat_idx] = default_flags[flat_idx] + + +@wp.kernel +def _kernel_body_particle_reaction( + contact_count: wp.array(dtype=wp.int32), + contact_particle: wp.array(dtype=wp.int32), + contact_shape: wp.array(dtype=wp.int32), + contact_body_pos: wp.array(dtype=wp.vec3), + contact_body_vel: wp.array(dtype=wp.vec3), + contact_normal: wp.array(dtype=wp.vec3), + particle_q: wp.array(dtype=wp.vec3), + particle_qd: wp.array(dtype=wp.vec3), + particle_radius: wp.array(dtype=wp.float32), + body_q: wp.array(dtype=wp.transform), + body_q_prev: wp.array(dtype=wp.transform), + body_qd: wp.array(dtype=wp.spatial_vector), + body_com: wp.array(dtype=wp.vec3), + shape_body: wp.array(dtype=wp.int32), + shape_material_mu: wp.array(dtype=wp.float32), + soft_contact_ke: float, + soft_contact_kd: float, + soft_contact_mu: float, + friction_epsilon: float, + dt: float, + body_f: wp.array(dtype=wp.spatial_vector), +): + """Newton's-third-law reaction from soft particles onto rigid bodies. + + Delegates to Newton's ``evaluate_body_particle_contact()`` for the contact + force computation (normal + damping + Coulomb friction) so the model stays + in sync with the VBD solver. The force on the particle is negated and + applied as a wrench on the rigid body via ``body_f``. + + One thread per contact slot; threads beyond the actual contact count + early-exit. + + The "previous" particle position required by the contact model is + reconstructed from the current velocity (``particle_q - particle_qd * dt``) + rather than read from a stored previous-state array. VBD mutates + ``particle_q`` in place during its iteration, so the swapped state's + ``particle_q`` is no longer a reliable snapshot of the prior substep. + """ + tid = wp.tid() + if tid >= contact_count[0]: + return + + p_idx = contact_particle[tid] + s_idx = contact_shape[tid] + body_idx = shape_body[s_idx] + if body_idx < 0: + return + + # Reconstruct previous particle position from velocity so that + # dx = particle_qd * dt regardless of what VBD wrote into stored states. + p_pos = particle_q[p_idx] + p_pos_prev = p_pos - particle_qd[p_idx] * dt + + # Delegate to Newton's canonical contact model + f_on_particle, _ = _evaluate_body_particle_contact( + p_idx, + p_pos, + p_pos_prev, + tid, + soft_contact_ke, + soft_contact_kd, + soft_contact_mu, + friction_epsilon, + particle_radius, + shape_material_mu, + shape_body, + body_q, + body_q_prev, + body_qd, + body_com, + contact_shape, + contact_body_pos, + contact_body_vel, + contact_normal, + dt, + ) + + # Newton's third law: negate particle force → rigid body wrench + X_wb = body_q[body_idx] + bx = wp.transform_point(X_wb, contact_body_pos[tid]) + com_w = wp.transform_point(X_wb, body_com[body_idx]) + + reaction = -f_on_particle + torque = wp.cross(bx - com_w, reaction) + + wp.atomic_add( + body_f, + body_idx, + wp.spatial_vector( + reaction[0], + reaction[1], + reaction[2], + torque[0], + torque[1], + torque[2], + ), + ) diff --git a/source/isaaclab_contrib/isaaclab_contrib/deformable/newton_manager_cfg.py b/source/isaaclab_contrib/isaaclab_contrib/deformable/newton_manager_cfg.py new file mode 100644 index 00000000000..e9358ff0be2 --- /dev/null +++ b/source/isaaclab_contrib/isaaclab_contrib/deformable/newton_manager_cfg.py @@ -0,0 +1,211 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Configuration classes for VBD, coupled solver, and global Newton model parameters.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from isaaclab_newton.physics import FeatherstoneSolverCfg, MJWarpSolverCfg, NewtonSolverCfg + +from isaaclab.utils.configclass import configclass + +if TYPE_CHECKING: + from isaaclab_newton.physics import NewtonManager + + +@configclass +class VBDSolverCfg(NewtonSolverCfg): + """Configuration for the Vertex Block Descent (VBD) solver. + + Supports particle simulation (cloth, soft bodies) and coupled rigid-body systems. + Requires ``ModelBuilder.color()`` to be called before ``finalize()`` to build + the parallel vertex colouring needed by the solver. + """ + + class_type: type[NewtonManager] | str = "{DIR}.vbd_manager:NewtonVBDManager" + """Manager class for the VBD solver.""" + + solver_type: str = "vbd" + + iterations: int = 10 + """Number of VBD iterations per substep.""" + + integrate_with_external_rigid_solver: bool = False + """Whether rigid bodies are integrated by an external solver (one-way coupling). + + Set to ``True`` when coupling cloth with a separate rigid-body solver + (e.g. ``SolverFeatherstone``) so that VBD only integrates the cloth particles. + """ + + particle_enable_self_contact: bool = False + """Whether to enable VBD deformable's self-contact.""" + + particle_self_contact_radius: float = 0.005 + """Particle radius used for self-contact detection [m].""" + + particle_self_contact_margin: float = 0.005 + """Self-contact detection margin [m]. Should be >= particle_self_contact_radius.""" + + particle_collision_detection_interval: int = -1 + """Controls how frequently particle self-contact detection is applied. + + If set to a value < 0, collision detection is only performed once before the + initialization step. If set to 0, collision detection is applied twice: once + before and once immediately after initialization. If set to a value ``k`` >= 1, + collision detection is applied before every ``k`` VBD iterations. + """ + + particle_vertex_contact_buffer_size: int = 32 + """Preallocation size for each vertex's vertex-triangle collision buffer.""" + + particle_edge_contact_buffer_size: int = 64 + """Preallocation size for each edge's edge-edge collision buffer.""" + + particle_topological_contact_filter_threshold: int = 2 + """Maximum topological distance (in rings) below which self-contacts are discarded. + + Only used when ``particle_enable_self_contact`` is ``True``. + Increase to suppress contacts between closely connected mesh elements. + Values > 3 significantly increase computation time. + """ + + particle_rest_shape_contact_exclusion_radius: float = 0.0 + """World-space distance threshold for filtering topologically close primitives [m]. + + Candidate self-contacts whose rest-configuration separation is shorter than + this value are ignored. Only used when ``particle_enable_self_contact`` is ``True``. + """ + + rigid_contact_k_start: float = 1.0e2 + """Initial stiffness seed for all rigid body contacts (body-body and body-particle) [N/m]. + + Used by the AVBD rigid contact solver. Increase to make rigid contacts stiffer. + """ + + +@configclass +class CoupledMJWarpVBDSolverCfg(NewtonSolverCfg): + """Configuration for the coupled rigid-body MJWarp + VBD solver. + + Alternates a rigid-body solver (:class:`MJWarpSolverCfg`) and a soft-body solver (:class:`SolverVBD`) per + substep. The coupling direction is controlled by :attr:`coupling_mode`: + + - ``"one_way"`` (default): Rigid solver advances first, then VBD reads + the updated body poses. The rigid solver does not feel particle contacts. + - ``"two_way"``: Same-substep two-way coupling with normal + Coulomb + friction. Contact detection runs first, reaction forces are injected + into ``body_f``, then the rigid solver reads ``body_f`` and feels + resistance from the deformable object. The friction reaction lets + actuators carry the object against gravity during a lift. + """ + + class_type: type[NewtonManager] | str = "{DIR}.coupled_mjwarp_vbd_manager:NewtonCoupledMJWarpVBDManager" + """Manager class for the VBD solver.""" + + solver_type: str = "coupledmjwarpvbd" + + rigid_solver_cfg: MJWarpSolverCfg = MJWarpSolverCfg() + """Rigid-body sub-solver configuration for :class:`MJWarpSolverCfg`.""" + + soft_solver_cfg: VBDSolverCfg = VBDSolverCfg(integrate_with_external_rigid_solver=True) + """VBD sub-solver configuration for cloth/particle dynamics.""" + + coupling_mode: str = "two_way" + """Coupling direction between the rigid and VBD solvers. + + - ``"one_way"``: Rigid -> soft only (default, existing behavior). + - ``"two_way"``: Same-substep two-way coupling with normal + Coulomb friction. + """ + + +@configclass +class CoupledFeatherstoneVBDSolverCfg(NewtonSolverCfg): + """Configuration for the coupled rigid-body Featherstone + VBD solver. + + Alternates a rigid-body solver (:class:`FeatherstoneSolverCfg`) and a soft-body solver (:class:`SolverVBD`) per + substep. The coupling direction is controlled by :attr:`coupling_mode`: + + - ``"kinematic"`` (default): Rigid -> soft only. Rigid bodies are kinematically updated by the rigid solver, + then VBD reads the updated body poses and reacts to them. The rigid solver does not feel particle contacts. + - ``"one_way"``: Rigid solver advances first, then VBD reads + the updated body poses. The rigid solver does not feel particle contacts. + - ``"two_way"``: Same-substep two-way coupling with normal + Coulomb + friction. Contact detection runs first, reaction forces are injected + into ``body_f``, then the rigid solver reads ``body_f`` and feels + resistance from the deformable object. The friction reaction lets + actuators carry the object against gravity during a lift. + """ + + class_type: type[NewtonManager] | str = "{DIR}.coupled_featherstone_vbd_manager:NewtonCoupledFeatherstoneVBDManager" + """Manager class for the VBD solver.""" + + solver_type: str = "coupledfeatherstonevbd" + + rigid_solver_cfg: FeatherstoneSolverCfg = FeatherstoneSolverCfg() + """Rigid-body sub-solver configuration for :class:`FeatherstoneSolverCfg`.""" + + soft_solver_cfg: VBDSolverCfg = VBDSolverCfg(integrate_with_external_rigid_solver=True) + """VBD sub-solver configuration for cloth/particle dynamics.""" + + coupling_mode: str = "kinematic" + """Coupling direction between the rigid and VBD solvers. + + - ``"kinematic"``: Rigid -> soft only (default) + - ``"one_way"``: Rigid -> soft only (existing behavior). + - ``"two_way"``: Same-substep two-way coupling with normal + Coulomb friction. + """ + + +@configclass +class NewtonModelCfg: + """Global Newton model parameters. + + These parameters are applied to the ``newton.Model`` after finalization. + They control model-level contact behavior shared across all objects. + """ + + soft_contact_ke: float = 1.0e3 + """Body-particle contact stiffness [N/m]. + + Controls the stiffness of the penalty force of contacts between cloth/soft-body particles + and rigid body shapes, and self-contacts of cloth/soft-body particles. The effective stiffness per contact is the + average of this value and the rigid shape's material stiffness. + """ + + soft_contact_kd: float = 1.0e-2 + """Body-particle contact damping [N*s/m].""" + + soft_contact_mu: float = 0.5 + """Body-particle contact friction coefficient. + + The effective friction per contact is ``sqrt(soft_contact_mu * shape_material_mu)``. + Increase for better grip (e.g. gripper picking up cloth). + """ + + shape_material_ke: float | None = None + """Per-shape contact stiffness override [N/m]. + + When set, all collision shapes in the model will have their contact + stiffness overwritten to this value. If ``None`` (default), the + per-shape values parsed from USD/MJCF are kept. + """ + + shape_material_kd: float | None = None + """Per-shape contact damping override [N*s/m]. + + When set, all collision shapes in the model will have their contact + damping overwritten to this value. If ``None`` (default), the + per-shape values parsed from USD/MJCF are kept. + """ + + shape_material_mu: float | None = None + """Per-shape friction coefficient override [dimensionless]. + + When set, all collision shapes in the model will have their friction + coefficient overwritten to this value. If ``None`` (default), the + per-shape values parsed from USD/MJCF are kept. + """ diff --git a/source/isaaclab_contrib/isaaclab_contrib/deformable/vbd_manager.py b/source/isaaclab_contrib/isaaclab_contrib/deformable/vbd_manager.py new file mode 100644 index 00000000000..88380f07867 --- /dev/null +++ b/source/isaaclab_contrib/isaaclab_contrib/deformable/vbd_manager.py @@ -0,0 +1,285 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""VBD Newton manager.""" + +from __future__ import annotations + +import inspect +import logging +from typing import TYPE_CHECKING + +import warp as wp +from isaaclab_newton.physics.newton_manager import NewtonManager +from newton import Model, ModelBuilder +from newton._src.usd.schemas import SchemaResolverNewton, SchemaResolverPhysx +from newton.solvers import SolverVBD + +from isaaclab.sim.utils.stage import get_current_stage + +from .deformable_object import ( + add_deformable_entry_to_builder, + clear_deformable_builder_hooks, + install_deformable_builder_hooks, +) +from .newton_manager_cfg import VBDSolverCfg + +if TYPE_CHECKING: + from isaaclab.sim.simulation_context import SimulationContext + +logger = logging.getLogger(__name__) + + +class NewtonVBDManager(NewtonManager): + """:class:`NewtonManager` specialization for the VBD solver. + + Always uses Newton's :class:`CollisionPipeline` for contact handling. + """ + + @classmethod + def initialize(cls, sim_context: SimulationContext) -> None: + """Initialize the manager with simulation context. + + Args: + sim_context: Parent simulation context. + + TODO: Subclass should not override this method, once deformables + supported on Newton import_usd, this can be unified with NewtonManager's + implementation. + """ + + # Deformable body registry and extension hooks. + # Experimental deformable support registers callbacks here so the manager + # and cloner can invoke them without hard-coding deformable logic. + install_deformable_builder_hooks() + + super().initialize(sim_context) + + @classmethod + def _solver_specific_clear(cls): + """Clear VBD-specific state.""" + clear_deformable_builder_hooks() + + @classmethod + def _get_deformable_ignore_paths(cls) -> list[str]: + """Return USD prim paths to skip when calling ``builder.add_usd``. + + For each registered deformable body, both the simulation mesh (which + carries ``UsdPhysics.CollisionAPI``) and the visual mesh are returned. + The sim mesh must be skipped so Newton does not create a redundant + static mesh collider alongside the particles produced by + ``add_soft_mesh``. The visual mesh is skipped so Newton does not + treat it as a collider — Kit reads it directly from USD for rendering. + + Paths may contain regex patterns; Newton's ``add_usd`` matches them + via :func:`re.match`. + """ + paths: list[str] = [] + for entry in cls._deformable_registry: + paths.append(entry.sim_mesh_prim_path) + paths.append(entry.vis_mesh_prim_path) + return paths + + @classmethod + def start_simulation(cls) -> None: + """Start simulation by finalizing model and initializing state. + + This function finalizes the model and initializes the simulation state. + Note: Collision pipeline is initialized later in initialize_solver() after + we determine whether the solver needs external collision detection. + + TODO: Subclass should not override this method, missing piece is + having Newton bind a surface mesh to volume deformable tetrahedral mesh + in addition to removing the deformable_registry data structure. + """ + super().start_simulation() + + # Apply global model parameters from :class:`NewtonModelCfg` to the finalized model. + # Sets ``soft_contact_ke/kd/mu`` and optionally overrides per-shape + # ``shape_material_ke/kd/mu`` on the Newton model. + from isaaclab.physics import PhysicsManager + + cfg = PhysicsManager._cfg + if cfg is not None and hasattr(cfg, "model_cfg") and cfg.model_cfg is not None: + model = cls._model + if model is None: + return + + model_cfg = cfg.model_cfg + model.soft_contact_ke = float(model_cfg.soft_contact_ke) + model.soft_contact_kd = float(model_cfg.soft_contact_kd) + model.soft_contact_mu = float(model_cfg.soft_contact_mu) + + if model_cfg.shape_material_ke is not None: + model.shape_material_ke.fill_(float(model_cfg.shape_material_ke)) + if model_cfg.shape_material_kd is not None: + model.shape_material_kd.fill_(float(model_cfg.shape_material_kd)) + if model_cfg.shape_material_mu is not None: + model.shape_material_mu.fill_(float(model_cfg.shape_material_mu)) + + # Setup USD/Fabric sync for Kit viewport deformable rendering + if not cls._clone_physics_only and cls._deformable_registry: + import re + + import usdrt + + if NewtonManager._usdrt_stage is None: + NewtonManager._usdrt_stage = get_current_stage(fabric=True) + + stage = get_current_stage() + for entry in cls._deformable_registry: + for inst_idx, offset in enumerate(entry.particle_offsets): + # Resolve regex pattern to concrete instance path of visual mesh + resolved_vis = re.sub(r"(?<=[Ee]nv_)\.\*", str(inst_idx), entry.vis_mesh_prim_path) + resolved_vis = re.sub(r"\.\*", str(inst_idx), resolved_vis) + vis_prim = stage.GetPrimAtPath(resolved_vis) + + if not vis_prim or not vis_prim.IsValid(): + logger.warning("[setup_fabric_particle_sync] vis prim not found at %s", resolved_vis) + continue + + # Create per-instance particle offset and count attributes on the visual mesh + # prim so the Fabric sync kernel can find the right slice of particle_q + # and iterate only over this body's particles (counts vary across bodies). + fab_prim = NewtonManager._usdrt_stage.GetPrimAtPath(vis_prim.GetPath().pathString) + fab_prim.CreateAttribute( + NewtonManager._newton_particle_offset_attr, usdrt.Sdf.ValueTypeNames.UInt, True + ) + fab_prim.GetAttribute(NewtonManager._newton_particle_offset_attr).Set(offset) + fab_prim.CreateAttribute( + NewtonManager._newton_particle_count_attr, usdrt.Sdf.ValueTypeNames.UInt, True + ) + fab_prim.GetAttribute(NewtonManager._newton_particle_count_attr).Set(entry.particles_per_body) + + cls._mark_particles_dirty() + cls.sync_particles_to_usd() + + @classmethod + def instantiate_builder_from_stage(cls): + """Create builder from USD stage with special treatment for deformable + bodies, as these are not read from USD yet. + + Detects env Xforms (e.g. ``/World/Env_0``, ``/World/Env_1``) and builds + each as a separate Newton world via ``begin_world``/``end_world``. + Falls back to a flat ``add_usd`` when no env Xforms are found. + + TODO: Subclass should not override this method, once deformables + supported on Newton import_usd, this can be unified with NewtonManager's + implementation. + """ + import re + + from pxr import UsdGeom + + stage = get_current_stage() + up_axis = UsdGeom.GetStageUpAxis(stage) + + # Scan /World children for env-like Xforms (Env_0, env_1, ...) + env_pattern = re.compile(r"^[Ee]nv_(\d+)$") + world_prim = stage.GetPrimAtPath("/World") + env_paths: list[tuple[int, str]] = [] + if world_prim and world_prim.IsValid(): + for child in world_prim.GetChildren(): + m = env_pattern.match(child.GetName()) + if m: + env_paths.append((int(m.group(1)), child.GetPath().pathString)) + env_paths.sort(key=lambda x: x[0]) + + builder = ModelBuilder(up_axis=up_axis) + + schema_resolvers = [SchemaResolverNewton(), SchemaResolverPhysx()] + + # Deformable sim/visual mesh paths must be skipped by ``add_usd`` + # so they don't get duplicated as static colliders. + deformable_ignore_paths = cls._get_deformable_ignore_paths() + + if not env_paths: + # No env Xforms — flat loading + builder.add_usd(stage, ignore_paths=deformable_ignore_paths, schema_resolvers=schema_resolvers) + + # Add deformable bodies from the registry (single world at origin). + for entry in cls._deformable_registry: + add_deformable_entry_to_builder(builder, entry, 0, [0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 1.0]) + else: + # Load everything except the env subtrees (ground plane, lights, etc.) + ignore_paths = [path for _, path in env_paths] + deformable_ignore_paths + builder.add_usd(stage, ignore_paths=ignore_paths, schema_resolvers=schema_resolvers) + + # Build a prototype from the first env (all envs assumed identical) + _, proto_path = env_paths[0] + proto = ModelBuilder(up_axis=up_axis) + proto.add_usd( + stage, + root_path=proto_path, + ignore_paths=deformable_ignore_paths, + schema_resolvers=schema_resolvers, + ) + + # Inject registered sites into the proto before replication + global_sites, proto_sites = cls._cl_inject_sites(builder, {proto_path: proto}) + global_site_map: dict[str, tuple[int, None]] = {label: (idx, None) for label, idx in global_sites.items()} + num_worlds = len(env_paths) + local_site_map: dict[str, list[list[int]]] = {} + site_entries = proto_sites.get(id(proto), {}) + + # Add each env as a separate Newton world + xform_cache = UsdGeom.XformCache() + for col, (_, env_path) in enumerate(env_paths): + builder.begin_world() + offset = builder.shape_count + world_xform = xform_cache.GetLocalToWorldTransform(stage.GetPrimAtPath(env_path)) + translation = world_xform.ExtractTranslation() + rotation = world_xform.ExtractRotationQuat() + pos = (translation[0], translation[1], translation[2]) + quat = ( + rotation.GetImaginary()[0], + rotation.GetImaginary()[1], + rotation.GetImaginary()[2], + rotation.GetReal(), + ) + builder.add_builder(proto, xform=wp.transform(pos, quat)) + for label, proto_shape_indices in site_entries.items(): + if label not in local_site_map: + local_site_map[label] = [[] for _ in range(num_worlds)] + for proto_shape_idx in proto_shape_indices: + local_site_map[label][col].append(offset + proto_shape_idx) + + # Add deformable bodies from the registry into this world. + for entry in cls._deformable_registry: + add_deformable_entry_to_builder(builder, entry, col, list(pos), quat) + + builder.end_world() + + NewtonManager._cl_site_index_map = { + **global_site_map, + **{label: (None, per_world) for label, per_world in local_site_map.items()}, + } + NewtonManager._num_envs = len(env_paths) + + # Call builder.color() if any deformable entries were added (required by VBD solver) + if cls._deformable_registry: + builder.color() + + cls.set_builder(builder) + + @classmethod + def _build_solver(cls, model: Model, solver_cfg: VBDSolverCfg) -> None: + """Construct :class:`SolverVBD` and populate the base-class slots. + + VBD always uses Newton's :class:`CollisionPipeline` and steps with + separate input/output states, so the flags are fixed. + """ + valid = set(inspect.signature(SolverVBD.__init__).parameters) - {"self", "model"} + kwargs = {k: v for k, v in solver_cfg.to_dict().items() if k in valid} + NewtonManager._solver = SolverVBD(model, **kwargs) + NewtonManager._use_single_state = False + NewtonManager._needs_collision_pipeline = True + + @classmethod + def _simulate_physics_only(cls) -> None: + # Rebuild BVH once per step for solvers that require it (e.g. VBD cloth). + if hasattr(cls._solver, "rebuild_bvh"): + cls._solver.rebuild_bvh(cls._state_0) + super()._simulate_physics_only() diff --git a/source/isaaclab_contrib/test/deformable/test_deformable_builder_hooks.py b/source/isaaclab_contrib/test/deformable/test_deformable_builder_hooks.py new file mode 100644 index 00000000000..4a9c03932c5 --- /dev/null +++ b/source/isaaclab_contrib/test/deformable/test_deformable_builder_hooks.py @@ -0,0 +1,100 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +import math + +import pytest +import warp as wp +from isaaclab_newton.sim.spawners.materials import NewtonDeformableMaterialCfg + +from isaaclab_contrib.deformable import DeformableObject, VBDSolverCfg +from isaaclab_contrib.deformable.deformable_object import DeformableRegistryEntry, add_deformable_entry_to_builder + + +class _FakeBuilder: + def __init__(self): + self.particle_count = 0 + self.cloth_meshes = [] + + def add_cloth_mesh(self, **kwargs) -> None: + self.cloth_meshes.append(kwargs) + self.particle_count += len(kwargs["vertices"]) + + +def _make_surface_entry() -> DeformableRegistryEntry: + half_sqrt = math.sqrt(0.5) + return DeformableRegistryEntry( + prim_path="/World/envs/env_.*/cloth", + sim_mesh_prim_path="/World/envs/env_.*/cloth/mesh", + vis_mesh_prim_path="/World/envs/env_.*/cloth/mesh", + vertices=[ + wp.vec3(0.0, 0.0, 0.0), + wp.vec3(1.0, 0.0, 0.0), + wp.vec3(0.0, 1.0, 0.0), + ], + indices=[0, 1, 2], + init_pos=(1.0, 0.0, 0.0), + init_rot=(0.0, 0.0, half_sqrt, half_sqrt), + deformable_type="surface", + ) + + +def _vec3_as_tuple(value) -> tuple[float, float, float]: + return (float(value[0]), float(value[1]), float(value[2])) + + +def test_deformable_package_exports_public_symbols(): + """Test that deformable symbols are exported from the package root.""" + assert DeformableObject.__name__ == "DeformableObject" + assert VBDSolverCfg.__name__ == "VBDSolverCfg" + + +def test_newton_material_defaults_match_registry_defaults(): + """Test that Newton material cfg defaults match the deformable registry defaults.""" + material_cfg = NewtonDeformableMaterialCfg() + + assert material_cfg.density == DeformableRegistryEntry.density + assert material_cfg.particle_radius == DeformableRegistryEntry.particle_radius + + +def test_builder_hook_applies_env_quaternion_to_deformable_entry(): + """Test that deformable builder placement honors the environment quaternion.""" + entry = _make_surface_entry() + builder = _FakeBuilder() + half_sqrt = math.sqrt(0.5) + + add_deformable_entry_to_builder( + builder, + entry, + env_idx=0, + env_position=[10.0, 20.0, 30.0], + env_rotation=[0.0, 0.0, half_sqrt, half_sqrt], + ) + + mesh = builder.cloth_meshes[0] + rotated_x_axis = wp.quat_rotate(mesh["rot"], wp.vec3(1.0, 0.0, 0.0)) + + assert _vec3_as_tuple(mesh["pos"]) == pytest.approx((10.0, 21.0, 30.0)) + assert _vec3_as_tuple(rotated_x_axis) == pytest.approx((-1.0, 0.0, 0.0), abs=1e-6) + assert entry.particle_offsets == [0] + assert entry.particles_per_body == 3 + + +def test_builder_hook_resets_entry_offsets_on_first_environment(): + """Test that repeated model rebuilds do not accumulate stale particle offsets.""" + entry = _make_surface_entry() + builder = _FakeBuilder() + identity = [0.0, 0.0, 0.0, 1.0] + + add_deformable_entry_to_builder(builder, entry, 0, [0.0, 0.0, 0.0], identity) + add_deformable_entry_to_builder(builder, entry, 1, [1.0, 0.0, 0.0], identity) + + assert entry.particle_offsets == [0, 3] + + rebuilt_builder = _FakeBuilder() + add_deformable_entry_to_builder(rebuilt_builder, entry, 0, [0.0, 0.0, 0.0], identity) + + assert entry.particle_offsets == [0] + assert entry.particles_per_body == 3 diff --git a/source/isaaclab_contrib/test/deformable/test_deformable_object.py b/source/isaaclab_contrib/test/deformable/test_deformable_object.py new file mode 100644 index 00000000000..d64cc39331b --- /dev/null +++ b/source/isaaclab_contrib/test/deformable/test_deformable_object.py @@ -0,0 +1,717 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +# ignore private usage of variables warning +# pyright: reportPrivateUsage=none + + +"""Launch Isaac Sim Simulator first.""" + +from isaaclab.app import AppLauncher + +# launch omniverse app +simulation_app = AppLauncher(headless=True).app + +"""Rest everything follows.""" + +import pytest +import torch +import warp as wp +from flaky import flaky +from isaaclab_newton.physics import NewtonCfg +from isaaclab_newton.sim.schemas import NewtonDeformableBodyPropertiesCfg +from isaaclab_newton.sim.spawners.materials import ( + NewtonDeformableBodyMaterialCfg, + NewtonSurfaceDeformableBodyMaterialCfg, +) + +import isaaclab.sim as sim_utils +import isaaclab.utils.math as math_utils +from isaaclab.assets import DeformableObject, DeformableObjectCfg +from isaaclab.sim import SimulationCfg, build_simulation_context + +from isaaclab_contrib.deformable.newton_manager_cfg import VBDSolverCfg + +NEWTON_VBD_CFG = SimulationCfg( + physics=NewtonCfg( + solver_cfg=VBDSolverCfg(iterations=3), + num_substeps=2, + ), +) + + +def _newton_sim_context(device="cuda:0", gravity_enabled=True): + """Helper to create a Newton VBD simulation context.""" + NEWTON_VBD_CFG.device = device + NEWTON_VBD_CFG.gravity = (0.0, 0.0, -9.81) if gravity_enabled else (0.0, 0.0, 0.0) + return build_simulation_context(device=device, sim_cfg=NEWTON_VBD_CFG, auto_add_lighting=True) + + +def generate_cubes_scene( + num_cubes: int = 1, + height: float = 1.0, + device: str = "cuda:0", +) -> DeformableObject: + """Generate a scene with deformable tet-mesh cubes. + + Args: + num_cubes: Number of cubes to generate. + height: Height of the cubes. + device: Device to use for the simulation. + + Returns: + The deformable object representing the cubes. + """ + origins = torch.tensor([(i * 1.0, 0, height) for i in range(num_cubes)]).to(device) + for i, origin in enumerate(origins): + sim_utils.create_prim(f"/World/env_{i}", "Xform", translation=origin) + + cube_object_cfg = DeformableObjectCfg( + prim_path="/World/env_.*/Cube", + spawn=sim_utils.MeshCuboidCfg( + size=(0.1, 0.1, 0.1), + deformable_props=NewtonDeformableBodyPropertiesCfg(), + visual_material=sim_utils.PreviewSurfaceCfg(diffuse_color=(0.2, 0.8, 0.2)), + physics_material=NewtonDeformableBodyMaterialCfg( + density=500.0, + k_mu=1e4, + k_lambda=1e4, + ), + ), + init_state=DeformableObjectCfg.InitialStateCfg( + pos=(0.0, 0.0, height), + rot=(1.0, 0.0, 0.0, 0.0), + ), + ) + cube_object = DeformableObject(cfg=cube_object_cfg) + return cube_object + + +def generate_cloth_scene( + num_cloths: int = 1, + height: float = 1.0, + device: str = "cuda:0", +) -> DeformableObject: + """Generate a scene with surface deformable cloth squares. + + Args: + num_cloths: Number of cloths to generate. + height: Height of the cloths. + device: Device to use for the simulation. + + Returns: + The deformable object representing the cloths. + """ + origins = torch.tensor([(i * 1.0, 0, height) for i in range(num_cloths)]).to(device) + for i, origin in enumerate(origins): + sim_utils.create_prim(f"/World/env_{i}", "Xform", translation=origin) + + cloth_object_cfg = DeformableObjectCfg( + prim_path="/World/env_.*/Cloth", + spawn=sim_utils.MeshRectangleCfg( + size=(0.2, 0.2), + resolution=(3, 3), + deformable_props=NewtonDeformableBodyPropertiesCfg(), + visual_material=sim_utils.PreviewSurfaceCfg(diffuse_color=(0.2, 0.2, 0.8)), + physics_material=NewtonSurfaceDeformableBodyMaterialCfg(density=0.02, particle_radius=0.005), + ), + init_state=DeformableObjectCfg.InitialStateCfg( + pos=(0.0, 0.0, height), + rot=(1.0, 0.0, 0.0, 0.0), + ), + ) + return DeformableObject(cfg=cloth_object_cfg) + + +def generate_cuboid_and_cylinder_scene(height: float = 1.0) -> tuple[DeformableObject, DeformableObject]: + """Generate two independent deformable assets with different mesh shapes.""" + sim_utils.create_prim("/World/env_0", "Xform", translation=(0.0, 0.0, 0.0)) + + cuboid_cfg = DeformableObjectCfg( + prim_path="/World/env_.*/Cuboid", + spawn=sim_utils.MeshCuboidCfg( + size=(0.16, 0.08, 0.12), + deformable_props=NewtonDeformableBodyPropertiesCfg(), + visual_material=sim_utils.PreviewSurfaceCfg(diffuse_color=(0.2, 0.8, 0.2)), + physics_material=NewtonDeformableBodyMaterialCfg( + density=500.0, + k_mu=1e4, + k_lambda=1e4, + ), + ), + init_state=DeformableObjectCfg.InitialStateCfg(pos=(0.0, 0.0, height)), + ) + cylinder_cfg = DeformableObjectCfg( + prim_path="/World/env_.*/Cylinder", + spawn=sim_utils.MeshCylinderCfg( + radius=0.06, + height=0.14, + deformable_props=NewtonDeformableBodyPropertiesCfg(), + visual_material=sim_utils.PreviewSurfaceCfg(diffuse_color=(0.8, 0.2, 0.2)), + physics_material=NewtonDeformableBodyMaterialCfg( + density=500.0, + k_mu=1e4, + k_lambda=1e4, + ), + ), + init_state=DeformableObjectCfg.InitialStateCfg(pos=(0.4, 0.0, height + 0.2)), + ) + return DeformableObject(cfg=cuboid_cfg), DeformableObject(cfg=cylinder_cfg) + + +@pytest.fixture +def sim(): + """Create Newton VBD simulation context.""" + with _newton_sim_context() as sim: + sim._app_control_on_stop_handle = None + yield sim + + +def test_initialization(sim): + """Test initialization of Newton deformable objects.""" + num_cubes = 2 + cube_object = generate_cubes_scene(num_cubes=num_cubes) + + sim.reset() + + assert cube_object.is_initialized + assert cube_object.num_instances == num_cubes + assert cube_object.max_sim_vertices_per_body > 0 + + particles_per_body = cube_object.max_sim_vertices_per_body + + # nodal_state_w: (N, V, 6) + nodal_state = cube_object.data.nodal_state_w.torch + assert nodal_state.shape == (num_cubes, particles_per_body, 6) + + # nodal_pos_w: (N, V, 3) + nodal_pos = cube_object.data.nodal_pos_w.torch + assert nodal_pos.shape == (num_cubes, particles_per_body, 3) + + # nodal_vel_w: (N, V, 3) + nodal_vel = cube_object.data.nodal_vel_w.torch + assert nodal_vel.shape == (num_cubes, particles_per_body, 3) + + # root_pos_w: (N, 3) + root_pos = cube_object.data.root_pos_w.torch + assert root_pos.shape == (num_cubes, 3) + + # root_vel_w: (N, 3) + root_vel = cube_object.data.root_vel_w.torch + assert root_vel.shape == (num_cubes, 3) + + +def test_surface_initialization_and_freefall(sim): + """Test initialization and stepping for surface deformable objects.""" + num_cloths = 2 + cloth_object = generate_cloth_scene(num_cloths=num_cloths, height=5.0) + + sim.reset() + + assert cloth_object.is_initialized + assert cloth_object.num_instances == num_cloths + assert cloth_object._deformable_type == "surface" + assert cloth_object.max_sim_vertices_per_body > 0 + assert cloth_object.data.nodal_pos_w.torch.shape == ( + num_cloths, + cloth_object.max_sim_vertices_per_body, + 3, + ) + + initial_root_z = cloth_object.data.root_pos_w.torch[:, 2].clone() + for _ in range(5): + sim.step() + cloth_object.update(sim.cfg.dt) + + assert torch.all(cloth_object.data.root_pos_w.torch[:, 2] < initial_root_z) + + +def test_set_nodal_state(sim): + """Test setting the state of the deformable object.""" + num_cubes = 2 + cube_object = generate_cubes_scene(num_cubes=num_cubes) + + sim.reset() + + for state_type_to_randomize in ["nodal_pos_w", "nodal_vel_w"]: + state_dict = { + "nodal_pos_w": torch.zeros_like(cube_object.data.nodal_pos_w.torch), + "nodal_vel_w": torch.zeros_like(cube_object.data.nodal_vel_w.torch), + } + + for _ in range(5): + state_dict[state_type_to_randomize] = torch.randn( + num_cubes, cube_object.max_sim_vertices_per_body, 3, device=sim.device + ) + + for _ in range(5): + nodal_state = torch.cat( + [ + state_dict["nodal_pos_w"], + state_dict["nodal_vel_w"], + ], + dim=-1, + ) + cube_object.write_nodal_state_to_sim_index(nodal_state) + + torch.testing.assert_close(cube_object.data.nodal_state_w.torch, nodal_state, rtol=1e-5, atol=1e-5) + + sim.step() + cube_object.update(sim.cfg.dt) + + +def test_write_partial_env_ids(sim): + """Test writing to a subset of environments using env_ids.""" + num_cubes = 2 + cube_object = generate_cubes_scene(num_cubes=num_cubes) + + sim.reset() + + particles_per_body = cube_object.max_sim_vertices_per_body + default_pos = cube_object.data.nodal_pos_w.torch.clone() + + # Write new positions only for env 0 + new_pos = torch.randn(1, particles_per_body, 3, device=sim.device) + cube_object.write_nodal_pos_to_sim_index(new_pos, env_ids=torch.tensor([0], device=sim.device)) + cube_object.update(sim.cfg.dt) + + read_pos = cube_object.data.nodal_pos_w.torch + + # env 0 should have new positions + torch.testing.assert_close(read_pos[0], new_pos[0], rtol=1e-5, atol=1e-5) + + # other envs should be unchanged + torch.testing.assert_close(read_pos[1:], default_pos[1:], rtol=1e-5, atol=1e-5) + + +def test_write_partial_velocity_env_ids(sim): + """Test writing nodal velocities to a subset of environments.""" + num_cubes = 4 + cube_object = generate_cubes_scene(num_cubes=num_cubes) + + sim.reset() + + particles_per_body = cube_object.max_sim_vertices_per_body + default_vel = cube_object.data.nodal_vel_w.torch.clone() + + env_ids = torch.tensor([1], device=sim.device) + new_vel = torch.full((1, particles_per_body, 3), 0.25, device=sim.device) + new_vel[..., 2] = 1.0 + cube_object.write_nodal_velocity_to_sim_index(new_vel, env_ids=env_ids) + cube_object.update(sim.cfg.dt) + + read_vel = cube_object.data.nodal_vel_w.torch + torch.testing.assert_close(read_vel[1], new_vel[0], rtol=1e-5, atol=1e-5) + torch.testing.assert_close(cube_object.data.root_vel_w.torch[1], new_vel[0].mean(dim=0), rtol=1e-5, atol=1e-5) + torch.testing.assert_close(read_vel[0], default_vel[0], rtol=1e-5, atol=1e-5) + if num_cubes > 2: + torch.testing.assert_close(read_vel[2:], default_vel[2:], rtol=1e-5, atol=1e-5) + + +def test_full_data_writes_selected_env(sim): + """Test full-sized write buffers with selected environment ids.""" + num_cubes = 3 + cube_object = generate_cubes_scene(num_cubes=num_cubes) + + sim.reset() + + particles_per_body = cube_object.max_sim_vertices_per_body + env_ids = torch.tensor([1], device=sim.device) + + default_pos = cube_object.data.nodal_pos_w.torch.clone() + full_pos = default_pos + torch.linspace(0.1, 0.3, num_cubes, device=sim.device).view(num_cubes, 1, 1) + cube_object.write_nodal_pos_to_sim_index(full_pos, env_ids=env_ids, full_data=True) + cube_object.update(sim.cfg.dt) + + read_pos = cube_object.data.nodal_pos_w.torch + torch.testing.assert_close(read_pos[1], full_pos[1], rtol=1e-5, atol=1e-5) + torch.testing.assert_close(read_pos[0], default_pos[0], rtol=1e-5, atol=1e-5) + torch.testing.assert_close(read_pos[2], default_pos[2], rtol=1e-5, atol=1e-5) + + default_vel = cube_object.data.nodal_vel_w.torch.clone() + full_vel = torch.zeros(num_cubes, particles_per_body, 3, device=sim.device) + full_vel[0, :, 0] = 0.5 + full_vel[1, :, 1] = 0.75 + full_vel[2, :, 2] = 1.0 + cube_object.write_nodal_velocity_to_sim_index(full_vel, env_ids=env_ids, full_data=True) + cube_object.update(sim.cfg.dt) + + read_vel = cube_object.data.nodal_vel_w.torch + torch.testing.assert_close(read_vel[1], full_vel[1], rtol=1e-5, atol=1e-5) + torch.testing.assert_close(read_vel[0], default_vel[0], rtol=1e-5, atol=1e-5) + torch.testing.assert_close(read_vel[2], default_vel[2], rtol=1e-5, atol=1e-5) + + default_targets = cube_object.data.nodal_kinematic_target.torch.clone() + full_targets = torch.zeros(num_cubes, particles_per_body, 4, device=sim.device) + full_targets[..., :3] = cube_object.data.default_nodal_state_w.torch[..., :3] + full_targets[..., 3] = 1.0 + full_targets[1, :, :3] += torch.tensor([0.0, 0.0, 0.1], device=sim.device) + full_targets[1, :, 3] = 0.0 + cube_object.write_nodal_kinematic_target_to_sim_index(full_targets, env_ids=env_ids, full_data=True) + + read_targets = cube_object.data.nodal_kinematic_target.torch + torch.testing.assert_close(read_targets[1], full_targets[1], rtol=1e-5, atol=1e-5) + torch.testing.assert_close(read_targets[0], default_targets[0], rtol=1e-5, atol=1e-5) + torch.testing.assert_close(read_targets[2], default_targets[2], rtol=1e-5, atol=1e-5) + + +def test_kinematic_target_partial_env_ids_with_warp_input(sim): + """Test indexed kinematic target writes with device-native input arrays.""" + num_cubes = 3 + cube_object = generate_cubes_scene(num_cubes=num_cubes) + + sim.reset() + + particles_per_body = cube_object.max_sim_vertices_per_body + env_ids = torch.tensor([2], device=sim.device) + + default_targets = cube_object.data.nodal_kinematic_target.torch.clone() + targets = torch.zeros(1, particles_per_body, 4, device=sim.device) + targets[0, :, :3] = cube_object.data.default_nodal_state_w.torch[2, :, :3] + targets[0, :, :3] += torch.tensor([0.0, 0.0, 0.1], device=sim.device) + targets[0, :, 3] = 0.0 + + cube_object.write_nodal_kinematic_target_to_sim_index( + wp.from_torch(targets.contiguous(), dtype=wp.vec4f), env_ids=env_ids + ) + + read_targets = cube_object.data.nodal_kinematic_target.torch + torch.testing.assert_close(read_targets[2], targets[0], rtol=1e-5, atol=1e-5) + torch.testing.assert_close(read_targets[0], default_targets[0], rtol=1e-5, atol=1e-5) + torch.testing.assert_close(read_targets[1], default_targets[1], rtol=1e-5, atol=1e-5) + + +def test_mask_writes_selected_env(sim): + """Test full-sized write buffers with selected environment masks.""" + num_cubes = 3 + cube_object = generate_cubes_scene(num_cubes=num_cubes) + + sim.reset() + + env_mask = wp.array([False, True, False], dtype=wp.bool, device=sim.device) + + default_state = cube_object.data.nodal_state_w.torch.clone() + full_state = default_state.clone() + full_state[:, :, :3] += torch.tensor([10.0, 10.0, 10.0], device=sim.device) + full_state[:, :, 3:] = 10.0 + full_state[1, :, :3] = default_state[1, :, :3] + torch.tensor([0.1, 0.2, 0.3], device=sim.device) + full_state[1, :, 3:] = torch.tensor([0.4, 0.5, 0.6], device=sim.device) + cube_object.write_nodal_state_to_sim_mask(full_state, env_mask=env_mask) + cube_object.update(sim.cfg.dt) + + read_state = cube_object.data.nodal_state_w.torch + torch.testing.assert_close(read_state[1], full_state[1], rtol=1e-5, atol=1e-5) + torch.testing.assert_close(read_state[0], default_state[0], rtol=1e-5, atol=1e-5) + torch.testing.assert_close(read_state[2], default_state[2], rtol=1e-5, atol=1e-5) + + pos_before = cube_object.data.nodal_pos_w.torch.clone() + full_pos = pos_before + torch.tensor([5.0, 5.0, 5.0], device=sim.device) + full_pos[1] = pos_before[1] + torch.tensor([0.0, -0.1, 0.2], device=sim.device) + cube_object.write_nodal_pos_to_sim_mask(full_pos, env_mask=env_mask) + cube_object.update(sim.cfg.dt) + + read_pos = cube_object.data.nodal_pos_w.torch + torch.testing.assert_close(read_pos[1], full_pos[1], rtol=1e-5, atol=1e-5) + torch.testing.assert_close(read_pos[0], pos_before[0], rtol=1e-5, atol=1e-5) + torch.testing.assert_close(read_pos[2], pos_before[2], rtol=1e-5, atol=1e-5) + + vel_before = cube_object.data.nodal_vel_w.torch.clone() + full_vel = torch.full_like(vel_before, 7.0) + full_vel[1] = torch.tensor([0.7, 0.8, 0.9], device=sim.device) + cube_object.write_nodal_velocity_to_sim_mask(full_vel, env_mask=env_mask) + cube_object.update(sim.cfg.dt) + + read_vel = cube_object.data.nodal_vel_w.torch + torch.testing.assert_close(read_vel[1], full_vel[1], rtol=1e-5, atol=1e-5) + torch.testing.assert_close(read_vel[0], vel_before[0], rtol=1e-5, atol=1e-5) + torch.testing.assert_close(read_vel[2], vel_before[2], rtol=1e-5, atol=1e-5) + + default_targets = cube_object.data.nodal_kinematic_target.torch.clone() + full_targets = default_targets.clone() + full_targets[:, :, :3] = 3.0 + full_targets[:, :, 3] = 0.0 + full_targets[1, :, :3] = read_pos[1] + torch.tensor([0.0, 0.0, 0.1], device=sim.device) + cube_object.write_nodal_kinematic_target_to_sim_mask(full_targets, env_mask=env_mask) + + read_targets = cube_object.data.nodal_kinematic_target.torch + torch.testing.assert_close(read_targets[1], full_targets[1], rtol=1e-5, atol=1e-5) + torch.testing.assert_close(read_targets[0], default_targets[0], rtol=1e-5, atol=1e-5) + torch.testing.assert_close(read_targets[2], default_targets[2], rtol=1e-5, atol=1e-5) + + +@pytest.mark.parametrize( + "num_cubes, randomize_pos, randomize_rot", + [ + (1, False, False), + (1, True, False), + (1, False, True), + (2, True, True), + ], +) +@flaky(max_runs=3, min_passes=1) +def test_set_nodal_state_with_applied_transform(num_cubes, randomize_pos, randomize_rot): + """Test setting the state of the deformable object with applied transform. + + Applies random position/rotation transforms to the default nodal state, + writes it to simulation, steps with no gravity, and verifies the mean + nodal position (root_pos_w) matches the expected transformed centroid. + """ + cfg = SimulationCfg( + physics=NewtonCfg( + solver_cfg=VBDSolverCfg(iterations=3), + num_substeps=2, + ), + ) + cfg.device = "cuda:0" + cfg.gravity = (0.0, 0.0, 0.0) + + with build_simulation_context(device="cuda:0", sim_cfg=cfg, auto_add_lighting=True) as sim: + sim._app_control_on_stop_handle = None + cube_object = generate_cubes_scene(num_cubes=num_cubes, height=5.0) + sim.reset() + + for _ in range(5): + nodal_state = cube_object.data.default_nodal_state_w.torch.clone() + mean_nodal_pos_default = nodal_state[..., :3].mean(dim=1) + + if randomize_pos: + pos_w = 0.5 * torch.rand(cube_object.num_instances, 3, device=sim.device) + pos_w[:, 2] += 0.5 + else: + pos_w = None + if randomize_rot: + quat_w = math_utils.random_orientation(cube_object.num_instances, device=sim.device) + else: + quat_w = None + + # transform_nodal_pos: center, rotate, translate, un-center + nodal_pos = nodal_state[..., :3] + mean_pos = nodal_pos.mean(dim=1, keepdim=True) + centered = nodal_pos - mean_pos + nodal_state[..., :3] = math_utils.transform_points(centered, pos_w, quat_w) + mean_pos + mean_nodal_pos_init = nodal_state[..., :3].mean(dim=1) + + if pos_w is None: + torch.testing.assert_close(mean_nodal_pos_init, mean_nodal_pos_default, rtol=1e-5, atol=1e-5) + else: + torch.testing.assert_close(mean_nodal_pos_init, mean_nodal_pos_default + pos_w, rtol=1e-5, atol=1e-5) + + cube_object.write_nodal_state_to_sim_index(nodal_state) + + for _ in range(50): + sim.step() + cube_object.update(sim.cfg.dt) + + torch.testing.assert_close(cube_object.data.root_pos_w.torch, mean_nodal_pos_init, rtol=1e-4, atol=1e-4) + + +def test_freefall_analytical(sim): + """Test that one step of free-fall matches the inertia target prediction. + + VBD computes an inertia target per substep (h = sub_dt):: + + v_new = v + g * h + x_new = x + v_new * h + + then optimizes elastic + contact potentials around it. For free-fall + (no contacts, negligible elastic forces over one step), the final + position equals the inertia target. + + Starting from rest (v_0 = 0) with N substeps of h = dt/N: + + substep 1: v_1 = g*h, dx_1 = g*h^2 (1 * g*h^2) + substep 2: v_2 = 2*g*h, dx_2 = 2*g*h^2 (2 * g*h^2) + ... + substep k: v_k = k*g*h, dx_k = k*g*h^2 (k * g*h^2) + + total dz = g*h^2 * (1 + 2 + ... + N) = g*h^2 * N*(N+1)/2 + + """ + g = -9.81 + dt = 1.0 / 60.0 + num_substeps = 2 + sub_dt = dt / num_substeps + expected_dz = g * sub_dt**2 * num_substeps * (num_substeps + 1) / 2 + + cube_object = generate_cubes_scene(num_cubes=1, height=5.0) + sim.reset() + + x0 = cube_object.data.nodal_pos_w.torch.clone() + sim.step() + cube_object.update(sim.cfg.dt) + x1 = cube_object.data.nodal_pos_w.torch + + dz = x1[..., 2] - x0[..., 2] + # Every vertex should have the same Z displacement under uniform gravity + torch.testing.assert_close(dz, torch.full_like(dz, expected_dz), rtol=1e-2, atol=1e-5) + + +def test_nodal_pos_reads_current_state_after_odd_substep_swap(): + """Test deformable reads use the current Newton state after state swapping.""" + cfg = SimulationCfg( + physics=NewtonCfg( + solver_cfg=VBDSolverCfg(iterations=3), + num_substeps=1, + use_cuda_graph=False, + ), + ) + cfg.device = "cuda:0" + cfg.gravity = (0.0, 0.0, -9.81) + + with build_simulation_context(device="cuda:0", sim_cfg=cfg, auto_add_lighting=True) as sim: + sim._app_control_on_stop_handle = None + cube_object = generate_cubes_scene(num_cubes=1, height=5.0) + + sim.reset() + + initial_pos = cube_object.data.nodal_pos_w.torch.clone() + sim.step() + cube_object.update(sim.cfg.dt) + + stepped_pos = cube_object.data.nodal_pos_w.torch + assert torch.all(stepped_pos[..., 2] < initial_pos[..., 2]) + + +def test_set_kinematic_targets(sim): + """Test setting kinematic targets for the deformable object. + + Env 0 is kinematically constrained (all vertices pinned at default positions, + flag=0). Other envs are free (flag=1) and fall under gravity. After several + steps, env 0 should stay in place while the others have fallen. + """ + num_cubes = 4 + cube_object = generate_cubes_scene(num_cubes=num_cubes, height=5.0) + + sim.reset() + + particles_per_body = cube_object.max_sim_vertices_per_body + default_state = cube_object.data.default_nodal_state_w.torch + default_pos = default_state[..., :3].clone() + + # Build kinematic target buffer: (N, V, 4) = [x, y, z, flag] + nodal_kinematic_targets = torch.zeros(num_cubes, particles_per_body, 4, device=sim.device) + + for _ in range(5): + # Restore default state + cube_object.write_nodal_state_to_sim_index(default_state) + + # Env 0: pin all vertices at default positions (flag=0 = kinematic) + nodal_kinematic_targets[0, :, :3] = default_pos[0] + nodal_kinematic_targets[0, :, 3] = 0.0 + + # Other envs: free (flag=1) + nodal_kinematic_targets[1:, :, :3] = default_pos[1:] + nodal_kinematic_targets[1:, :, 3] = 1.0 + + cube_object.write_nodal_kinematic_target_to_sim_index(nodal_kinematic_targets) + + for _ in range(20): + cube_object.write_data_to_sim() + sim.step() + cube_object.update(sim.cfg.dt) + + # Env 0 should stay at default position (kinematically constrained) + torch.testing.assert_close( + cube_object.data.nodal_pos_w.torch[0], + default_pos[0], + rtol=1e-5, + atol=1e-5, + ) + + # Other envs should have fallen + final_root_z = cube_object.data.root_pos_w.torch[1:, 2] + default_root_z = default_pos[1:, :, 2].mean(dim=1) + assert torch.all(final_root_z < default_root_z) + + +def test_kinematic_target_release_restores_free_motion(sim): + """Test that a pinned deformable falls again after kinematic targets are released.""" + cube_object = generate_cubes_scene(num_cubes=1, height=5.0) + + sim.reset() + + particles_per_body = cube_object.max_sim_vertices_per_body + default_state = cube_object.data.default_nodal_state_w.torch + default_pos = default_state[..., :3].clone() + + targets = torch.zeros(1, particles_per_body, 4, device=sim.device) + targets[0, :, :3] = default_pos[0] + targets[0, :, 3] = 0.0 + cube_object.write_nodal_kinematic_target_to_sim_index(targets) + + for _ in range(5): + cube_object.write_data_to_sim() + sim.step() + cube_object.update(sim.cfg.dt) + + torch.testing.assert_close(cube_object.data.nodal_pos_w.torch[0], default_pos[0], rtol=1e-5, atol=1e-5) + + targets[0, :, 3] = 1.0 + cube_object.write_nodal_kinematic_target_to_sim_index(targets) + + for _ in range(20): + cube_object.write_data_to_sim() + sim.step() + cube_object.update(sim.cfg.dt) + + assert cube_object.data.root_pos_w.torch[0, 2] < default_pos[0, :, 2].mean() + + +def test_multiple_deformable_assets_do_not_alias(sim): + """Test independent writes for two different deformable assets in one scene.""" + cuboid, cylinder = generate_cuboid_and_cylinder_scene(height=2.0) + + sim.reset() + + cuboid_default = cuboid.data.nodal_pos_w.torch.clone() + cylinder_default = cylinder.data.nodal_pos_w.torch.clone() + + cuboid_pos = cuboid_default + torch.tensor([0.15, -0.05, 0.1], device=sim.device) + cuboid.write_nodal_pos_to_sim_index(cuboid_pos) + cuboid.update(sim.cfg.dt) + cylinder.update(sim.cfg.dt) + + torch.testing.assert_close(cuboid.data.nodal_pos_w.torch, cuboid_pos, rtol=1e-5, atol=1e-5) + torch.testing.assert_close(cylinder.data.nodal_pos_w.torch, cylinder_default, rtol=1e-5, atol=1e-5) + assert cuboid._recorded_particle_offsets != cylinder._recorded_particle_offsets + + +def test_rebind_after_sim_reset(sim): + """Test that deformable write paths remain valid after a simulation reset.""" + cube_object = generate_cubes_scene(num_cubes=1, height=2.0) + + sim.reset() + + initial_pos = cube_object.data.default_nodal_state_w.torch[..., :3].clone() + first_pos = initial_pos + torch.tensor([0.1, 0.0, 0.0], device=sim.device) + cube_object.write_nodal_pos_to_sim_index(first_pos) + cube_object.update(sim.cfg.dt) + torch.testing.assert_close(cube_object.data.nodal_pos_w.torch, first_pos, rtol=1e-5, atol=1e-5) + + sim.reset() + + second_pos = initial_pos + torch.tensor([0.0, -0.1, 0.2], device=sim.device) + cube_object.write_nodal_pos_to_sim_index(second_pos) + cube_object.update(sim.cfg.dt) + torch.testing.assert_close(cube_object.data.nodal_pos_w.torch, second_pos, rtol=1e-5, atol=1e-5) + + +def test_write_shape_validation(sim): + """Test public write APIs reject wrong tensor shapes.""" + cube_object = generate_cubes_scene(num_cubes=2) + + sim.reset() + + particles_per_body = cube_object.max_sim_vertices_per_body + wrong_pos = torch.zeros(particles_per_body, 3, device=sim.device) + wrong_vel = torch.zeros(1, particles_per_body, 2, device=sim.device) + wrong_targets = torch.zeros(2, particles_per_body, 3, device=sim.device) + + with pytest.raises(AssertionError, match="Shape mismatch"): + cube_object.write_nodal_pos_to_sim_index(wrong_pos) + with pytest.raises(AssertionError, match="Shape mismatch"): + cube_object.write_nodal_velocity_to_sim_index(wrong_vel, env_ids=torch.tensor([0], device=sim.device)) + with pytest.raises(AssertionError, match="Shape mismatch"): + cube_object.write_nodal_kinematic_target_to_sim_index(wrong_targets) diff --git a/source/isaaclab_contrib/test/deformable/test_rigid_deformable_coupling.py b/source/isaaclab_contrib/test/deformable/test_rigid_deformable_coupling.py new file mode 100644 index 00000000000..4b00c0d2e37 --- /dev/null +++ b/source/isaaclab_contrib/test/deformable/test_rigid_deformable_coupling.py @@ -0,0 +1,332 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +# ignore private usage of variables warning +# pyright: reportPrivateUsage=none + + +"""Launch Isaac Sim Simulator first.""" + +from isaaclab.app import AppLauncher + +# launch omniverse app +simulation_app = AppLauncher(headless=True).app + +"""Rest everything follows.""" + +import pytest +import torch +from isaaclab_newton.assets import Articulation, RigidObject +from isaaclab_newton.physics import FeatherstoneSolverCfg, MJWarpSolverCfg, NewtonCfg +from isaaclab_newton.sim.schemas import NewtonDeformableBodyPropertiesCfg +from isaaclab_newton.sim.spawners.materials import NewtonDeformableBodyMaterialCfg + +import isaaclab.sim as sim_utils +from isaaclab.assets import RigidObjectCfg +from isaaclab.assets.deformable_object import DeformableObjectCfg +from isaaclab.sim import SimulationCfg, build_simulation_context + +from isaaclab_contrib.deformable import ( + CoupledFeatherstoneVBDSolverCfg, + CoupledMJWarpVBDSolverCfg, + DeformableObject, + VBDSolverCfg, +) + +from isaaclab_assets import FRANKA_PANDA_CFG # isort:skip + + +def _make_coupled_cfg(coupling_mode: str, rigid_solver: str = "mjwarp") -> SimulationCfg: + """Create a simulation config for a coupled rigid-deformable solver.""" + if rigid_solver == "mjwarp": + solver_cfg = CoupledMJWarpVBDSolverCfg( + rigid_solver_cfg=MJWarpSolverCfg( + njmax=40, + nconmax=20, + ls_iterations=20, + cone="pyramidal", + impratio=1, + ls_parallel=False, + integrator="implicitfast", + ), + soft_solver_cfg=VBDSolverCfg( + iterations=3, + integrate_with_external_rigid_solver=True, + particle_enable_self_contact=False, + particle_collision_detection_interval=-1, + ), + coupling_mode=coupling_mode, + ) + elif rigid_solver == "featherstone": + solver_cfg = CoupledFeatherstoneVBDSolverCfg( + rigid_solver_cfg=FeatherstoneSolverCfg(), + soft_solver_cfg=VBDSolverCfg( + iterations=3, + integrate_with_external_rigid_solver=True, + particle_enable_self_contact=False, + particle_collision_detection_interval=-1, + ), + coupling_mode=coupling_mode, + ) + else: + raise ValueError(f"Unknown rigid solver: {rigid_solver}") + + return SimulationCfg( + dt=1.0 / 60.0, + physics=NewtonCfg( + solver_cfg=solver_cfg, + num_substeps=5, + use_cuda_graph=True, + ), + ) + + +def _coupled_sim_context(cfg: SimulationCfg, device="cuda:0"): + """Helper to create a coupled solver simulation context.""" + cfg.device = device + return build_simulation_context(device=device, sim_cfg=cfg, auto_add_lighting=True) + + +@pytest.fixture +def sim(request): + """Create a coupled solver simulation context. + + Defaults to one-way coupling. Tests can parametrize this fixture with + ``"two_way"`` when both coupling paths should be exercised. + """ + param = getattr(request, "param", "one_way") + if isinstance(param, tuple): + rigid_solver, coupling_mode = param + else: + rigid_solver, coupling_mode = "mjwarp", param + with _coupled_sim_context(_make_coupled_cfg(coupling_mode, rigid_solver)) as sim: + sim._app_control_on_stop_handle = None + yield sim + + +def generate_robot_and_two_cubes( + colliding_cube_pos: tuple = (0.3, 0.0, 1.0), + free_cube_pos: tuple = (2.0, 0.0, 1.0), +) -> tuple[Articulation, DeformableObject, DeformableObject]: + """Generate a scene with one Franka robot and two deformable cubes. + + A single env contains a robot and two cube objects at different positions. + One cube is placed above the robot arm (will collide), the other is placed + far away (falls freely). + + Args: + colliding_cube_pos: Position of the cube above the robot arm. + free_cube_pos: Position of the cube that falls freely. + + Returns: + Tuple of (robot, colliding_cube, free_cube). + """ + sim_utils.create_prim("/World/env_0", "Xform", translation=(0.0, 0.0, 0.0)) + + cfg = sim_utils.GroundPlaneCfg() + cfg.func("/World/defaultGroundPlane", cfg) + + robot_cfg = FRANKA_PANDA_CFG.replace(prim_path="/World/env_.*/Robot") + robot = Articulation(robot_cfg) + + colliding_cube = DeformableObject( + cfg=DeformableObjectCfg( + prim_path="/World/env_.*/cube_collide", + spawn=sim_utils.MeshCuboidCfg( + size=(0.05, 0.05, 0.05), + deformable_props=NewtonDeformableBodyPropertiesCfg(), + visual_material=sim_utils.PreviewSurfaceCfg(diffuse_color=(0.2, 0.8, 0.2)), + physics_material=NewtonDeformableBodyMaterialCfg( + density=500.0, + k_mu=1e5, + k_lambda=1e5, + particle_radius=0.005, + ), + ), + init_state=DeformableObjectCfg.InitialStateCfg(pos=colliding_cube_pos), + ) + ) + + free_cube = DeformableObject( + cfg=DeformableObjectCfg( + prim_path="/World/env_.*/cube_free", + spawn=sim_utils.MeshCuboidCfg( + size=(0.05, 0.05, 0.05), + deformable_props=NewtonDeformableBodyPropertiesCfg(), + visual_material=sim_utils.PreviewSurfaceCfg(diffuse_color=(0.8, 0.2, 0.2)), + physics_material=NewtonDeformableBodyMaterialCfg( + density=500.0, + k_mu=1e4, + k_lambda=1e4, + particle_radius=0.005, + ), + ), + init_state=DeformableObjectCfg.InitialStateCfg(pos=free_cube_pos), + ) + ) + + return robot, colliding_cube, free_cube + + +def generate_lateral_rigid_and_deformable_cubes( + rigid_cube_pos: tuple = (0.0, 0.0, 1.0), + deformable_cube_pos: tuple = (-0.16, 0.0, 1.0), +) -> tuple[RigidObject, DeformableObject]: + """Generate rigid and deformable cubes arranged for lateral contact. + + Args: + rigid_cube_pos: Initial position of the rigid cube. + deformable_cube_pos: Initial position of the deformable cube. + + Returns: + Tuple of (rigid cube, deformable cube). + """ + sim_utils.create_prim("/World/env_0", "Xform", translation=(0.0, 0.0, 0.0)) + + rigid_cube = RigidObject( + cfg=RigidObjectCfg( + prim_path="/World/env_.*/rigid_cube", + spawn=sim_utils.CuboidCfg( + size=(0.2, 0.2, 0.2), + rigid_props=sim_utils.RigidBodyPropertiesCfg(), + mass_props=sim_utils.MassPropertiesCfg(mass=0.05), + collision_props=sim_utils.CollisionPropertiesCfg(), + visual_material=sim_utils.PreviewSurfaceCfg(diffuse_color=(0.2, 0.2, 0.8)), + ), + init_state=RigidObjectCfg.InitialStateCfg(pos=rigid_cube_pos), + ) + ) + + deformable_cube = DeformableObject( + cfg=DeformableObjectCfg( + prim_path="/World/env_.*/deformable_cube", + spawn=sim_utils.MeshCuboidCfg( + size=(0.08, 0.08, 0.08), + deformable_props=NewtonDeformableBodyPropertiesCfg(), + visual_material=sim_utils.PreviewSurfaceCfg(diffuse_color=(0.8, 0.2, 0.2)), + physics_material=NewtonDeformableBodyMaterialCfg( + density=1000.0, + k_mu=1e5, + k_lambda=1e5, + particle_radius=0.005, + ), + ), + init_state=DeformableObjectCfg.InitialStateCfg(pos=deformable_cube_pos), + ) + ) + + return rigid_cube, deformable_cube + + +@pytest.mark.parametrize( + "sim", + [("featherstone", "kinematic")], + indirect=True, + ids=["featherstone_kinematic"], +) +def test_smoke_featherstone_kinematic(sim): + """Smoke test: Featherstone + VBD kinematic coupling initializes and steps.""" + robot, colliding_cube, free_cube = generate_robot_and_two_cubes() + sim.reset() + + assert robot.is_initialized + assert colliding_cube.is_initialized + assert free_cube.is_initialized + + initial_z_collide = colliding_cube.data.root_pos_w.torch[0, 2].item() + initial_z_free = free_cube.data.root_pos_w.torch[0, 2].item() + + for _ in range(10): + sim.step() + robot.update(sim.cfg.dt) + colliding_cube.update(sim.cfg.dt) + free_cube.update(sim.cfg.dt) + + assert colliding_cube.data.root_pos_w.torch[0, 2].item() < initial_z_collide - 0.01 + assert free_cube.data.root_pos_w.torch[0, 2].item() < initial_z_free - 0.01 + + +def _run_lateral_rigid_cube_response(coupling_mode: str) -> float: + """Run a compact lateral contact scene and return rigid cube X displacement.""" + with _coupled_sim_context(_make_coupled_cfg(coupling_mode)) as sim: + sim._app_control_on_stop_handle = None + rigid_cube, deformable_cube = generate_lateral_rigid_and_deformable_cubes() + sim.reset() + + initial_rigid_x = rigid_cube.data.root_pos_w.torch[0, 0].item() + nodal_vel = torch.zeros_like(deformable_cube.data.nodal_vel_w.torch) + nodal_vel[..., 0] = 2.0 + deformable_cube.write_nodal_velocity_to_sim_index(nodal_vel) + + for _ in range(60): + sim.step() + rigid_cube.update(sim.cfg.dt) + deformable_cube.update(sim.cfg.dt) + + return rigid_cube.data.root_pos_w.torch[0, 0].item() - initial_rigid_x + + +def test_two_way_coupling_applies_reaction_to_rigid_body(): + """Test that two-way coupling laterally pushes a rigid body.""" + one_way_dx = _run_lateral_rigid_cube_response("one_way") + two_way_dx = _run_lateral_rigid_cube_response("two_way") + + assert abs(one_way_dx) < 1e-2 + assert two_way_dx > one_way_dx + 1e-2 + + +def test_deformable_deflected_by_rigid_contact(sim): + """Test that a cube falling onto the robot is deflected horizontally. + + Two cubes start at the same height (Z=1.0m). Cube 0 (env 0) is placed + above the robot arm at X=0.3m. Cube 1 (env 1) is shifted +2m in X away + from the robot so it falls freely. + + Expected timeline (dt=1/60s): + + - Steps 0-15: Both cubes in free-fall, identical Z trajectories. + No horizontal motion. Z drops from ~1.0m to ~0.65m. + - Step ~20: Cube 0 hits the robot arm at Z~0.54m. Contact deflects it + horizontally (X starts increasing). Cube 1 is still in free-fall. + - Step ~28: Cube 1 reaches the ground (Z~0.005m) and stops. Zero + horizontal displacement. + - Steps 20-40: Cube 0 continues falling while sliding off the arm, + gaining horizontal velocity. Reaches the ground around step 40. + - Steps 40-70: Cube 0 slides along the ground, decelerating due to + friction. Settles around X~1.0m. + - Steps 70+: Both cubes at rest on the ground. + + The test asserts that cube 0 has a significantly larger horizontal + displacement than cube 1 (which should be ~zero). + """ + robot, colliding_cube, free_cube = generate_robot_and_two_cubes( + colliding_cube_pos=(0.3, 0.0, 1.0), # above robot arm + free_cube_pos=(2.0, 0.0, 1.0), # far from robot, same height + ) + sim.reset() + + initial_xy_collide = colliding_cube.data.root_pos_w.torch[0, :2].clone() + initial_xy_free = free_cube.data.root_pos_w.torch[0, :2].clone() + + # Free-fall from 1.0m takes sqrt(2*1.0/9.81) ~ 0.45s ~ 27 steps at dt=1/60. + # Collision with the robot arm happens around step 20 (Z~0.5m). + # 120 steps (2s) gives ample time for collision, bounce, and settling. + for _ in range(120): + sim.step() + robot.update(sim.cfg.dt) + colliding_cube.update(sim.cfg.dt) + free_cube.update(sim.cfg.dt) + + final_xy_collide = colliding_cube.data.root_pos_w.torch[0, :2] + final_xy_free = free_cube.data.root_pos_w.torch[0, :2] + + displacement_collide = (final_xy_collide - initial_xy_collide).norm().item() + displacement_free = (final_xy_free - initial_xy_free).norm().item() + + # Colliding cube should be deflected; free cube should fall straight + assert displacement_collide > displacement_free + 0.01, ( + f"Colliding cube should be deflected more than free cube: " + f"collide={displacement_collide:.4f}, free={displacement_free:.4f}" + ) diff --git a/source/isaaclab_newton/changelog.d/daniela-move-scene-data.rst b/source/isaaclab_newton/changelog.d/daniela-move-scene-data.rst new file mode 100644 index 00000000000..1c077dc3f68 --- /dev/null +++ b/source/isaaclab_newton/changelog.d/daniela-move-scene-data.rst @@ -0,0 +1,14 @@ +Changed +^^^^^^^ + +* Updated imports of :class:`~isaaclab.scene_data.SceneDataBackend` and + :class:`~isaaclab.scene_data.SceneDataFormat` to their new location in + :mod:`isaaclab.scene_data` (previously :mod:`isaaclab.physics`). + +Fixed +^^^^^ + +* Fixed :meth:`~isaaclab_newton.physics.NewtonManager.update_visualization_state` + retrieving the wrong simulation context. It now uses + :meth:`~isaaclab.sim.SimulationContext.instance` instead of the stale + ``PhysicsManager._sim`` reference. diff --git a/source/isaaclab_newton/changelog.d/mtrepte-tiled_viz.rst b/source/isaaclab_newton/changelog.d/mtrepte-tiled_viz.rst new file mode 100644 index 00000000000..96ea88dc916 --- /dev/null +++ b/source/isaaclab_newton/changelog.d/mtrepte-tiled_viz.rst @@ -0,0 +1,4 @@ +Fixed +^^^^^ + +* Fixed Newton visualizer camera image views and state updates for PhysX-backed simulations. diff --git a/source/isaaclab_newton/changelog.d/mtrepte-update-debug-viz.rst b/source/isaaclab_newton/changelog.d/mtrepte-update-debug-viz.rst new file mode 100644 index 00000000000..2ca3ab37422 --- /dev/null +++ b/source/isaaclab_newton/changelog.d/mtrepte-update-debug-viz.rst @@ -0,0 +1,4 @@ +Fixed +^^^^^ + +* Fixed Newton visualization state updates for PhysX-backed simulations. diff --git a/source/isaaclab_newton/changelog.d/mym-deformable-backend-split.minor.rst b/source/isaaclab_newton/changelog.d/mym-deformable-backend-split.minor.rst new file mode 100644 index 00000000000..3dade7d6fb9 --- /dev/null +++ b/source/isaaclab_newton/changelog.d/mym-deformable-backend-split.minor.rst @@ -0,0 +1,4 @@ +Added +^^^^^ + +* Added Newton-specific deformable property and material cfgs. diff --git a/source/isaaclab_newton/changelog.d/mym-deformable_experimental.minor.rst b/source/isaaclab_newton/changelog.d/mym-deformable_experimental.minor.rst new file mode 100644 index 00000000000..aacba4481af --- /dev/null +++ b/source/isaaclab_newton/changelog.d/mym-deformable_experimental.minor.rst @@ -0,0 +1,33 @@ +Added +^^^^^ + +* Added Newton deformable asset exports under + :mod:`isaaclab_newton.assets.deformable_object`. +* Added deformable registration hooks to Newton cloning so deformable assets can + be added per replicated world while their USD proxy meshes are skipped by the + Newton USD importer. +* Added Newton manager abstraction documentation for adding solver managers and + custom coupled solvers. + +Changed +^^^^^^^ + +* Changed Newton solver configuration exports so + :class:`~isaaclab_newton.physics.MJWarpSolverCfg`, + :class:`~isaaclab_newton.physics.XPBDSolverCfg`, + :class:`~isaaclab_newton.physics.FeatherstoneSolverCfg`, and + :class:`~isaaclab_newton.physics.KaminoSolverCfg` are provided from + :mod:`isaaclab_newton.physics.newton_manager_cfg`. +* Changed :class:`~isaaclab_newton.physics.NewtonCfg` to use + :class:`~isaaclab_newton.physics.MJWarpSolverCfg` as its explicit default + solver configuration. +* Changed :class:`~isaaclab_newton.physics.NewtonCfg` validation to reject + :class:`~isaaclab_newton.physics.MJWarpSolverCfg` configurations that combine + ``use_mujoco_contacts=True`` with ``collision_cfg``. Remove ``collision_cfg`` + or set ``use_mujoco_contacts=False``. + +Fixed +^^^^^ + +* Fixed Newton Fabric synchronization for deformable particle meshes and + particle-only scenes. diff --git a/source/isaaclab_newton/docs/CHANGELOG.rst b/source/isaaclab_newton/docs/CHANGELOG.rst index 836ba4a2c13..03ceddc8200 100644 --- a/source/isaaclab_newton/docs/CHANGELOG.rst +++ b/source/isaaclab_newton/docs/CHANGELOG.rst @@ -116,7 +116,7 @@ Added USD stage (via :meth:`~isaaclab_newton.physics.NewtonManager.instantiate_builder_from_stage`) and refreshes ``state_0.body_q`` from rigid-body transforms supplied by the - :class:`~isaaclab.scene.scene_data_provider.SceneDataProvider` each render + :class:`~isaaclab.scene_data.SceneDataProvider` each render frame. Changed @@ -149,7 +149,7 @@ Removed (``NewtonSceneDataProvider``). Replace direct uses with :meth:`~isaaclab_newton.physics.NewtonManager.get_model` / :meth:`~isaaclab_newton.physics.NewtonManager.get_state` and the - Warp-native :class:`~isaaclab.scene.scene_data_provider.SceneDataProvider`. + Warp-native :class:`~isaaclab.scene_data.SceneDataProvider`. Fixed ^^^^^ diff --git a/source/isaaclab_newton/isaaclab_newton/assets/__init__.pyi b/source/isaaclab_newton/isaaclab_newton/assets/__init__.pyi index b9359445960..5476609e4ea 100644 --- a/source/isaaclab_newton/isaaclab_newton/assets/__init__.pyi +++ b/source/isaaclab_newton/isaaclab_newton/assets/__init__.pyi @@ -6,6 +6,8 @@ __all__ = [ "Articulation", "ArticulationData", + "DeformableObject", + "DeformableObjectData", "RigidObject", "RigidObjectData", "RigidObjectCollection", @@ -15,3 +17,4 @@ __all__ = [ from .articulation import Articulation, ArticulationData from .rigid_object import RigidObject, RigidObjectData from .rigid_object_collection import RigidObjectCollection, RigidObjectCollectionData +from .deformable_object import DeformableObject, DeformableObjectData diff --git a/source/isaaclab_newton/isaaclab_newton/assets/articulation/articulation.py b/source/isaaclab_newton/isaaclab_newton/assets/articulation/articulation.py index 69006598094..f921001cb8c 100644 --- a/source/isaaclab_newton/isaaclab_newton/assets/articulation/articulation.py +++ b/source/isaaclab_newton/isaaclab_newton/assets/articulation/articulation.py @@ -712,6 +712,7 @@ def write_root_velocity_to_sim_index( """Set the root center of mass velocity over selected environment indices into the simulation. The velocity comprises linear velocity (x, y, z) and angular velocity (x, y, z) in that order. + .. note:: This sets the velocity of the root's center of mass rather than the root's frame. @@ -738,6 +739,7 @@ def write_root_velocity_to_sim_mask( """Set the root center of mass velocity over selected environment mask into the simulation. The velocity comprises linear velocity (x, y, z) and angular velocity (x, y, z) in that order. + .. note:: This sets the velocity of the root's center of mass rather than the root's frame. @@ -764,6 +766,7 @@ def write_root_com_velocity_to_sim_index( """Set the root center of mass velocity over selected environment indices into the simulation. The velocity comprises linear velocity (x, y, z) and angular velocity (x, y, z) in that order. + .. note:: This sets the velocity of the root's center of mass rather than the root's frame. @@ -812,6 +815,7 @@ def write_root_com_velocity_to_sim_mask( """Set the root center of mass velocity over selected environment mask into the simulation. The velocity comprises linear velocity (x, y, z) and angular velocity (x, y, z) in that order. + .. note:: This sets the velocity of the root's center of mass rather than the root's frame. @@ -858,6 +862,7 @@ def write_root_link_velocity_to_sim_index( """Set the root link velocity over selected environment indices into the simulation. The velocity comprises linear velocity (x, y, z) and angular velocity (x, y, z) in that order. + .. note:: This sets the velocity of the root's frame rather than the root's center of mass. @@ -912,6 +917,7 @@ def write_root_link_velocity_to_sim_mask( """Set the root link velocity over selected environment mask into the simulation. The velocity comprises linear velocity (x, y, z) and angular velocity (x, y, z) in that order. + .. note:: This sets the velocity of the root's frame rather than the root's center of mass. diff --git a/source/isaaclab_newton/isaaclab_newton/assets/deformable_object/__init__.py b/source/isaaclab_newton/isaaclab_newton/assets/deformable_object/__init__.py new file mode 100644 index 00000000000..e14e0f6d52c --- /dev/null +++ b/source/isaaclab_newton/isaaclab_newton/assets/deformable_object/__init__.py @@ -0,0 +1,8 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +from isaaclab.utils.module import lazy_export + +lazy_export() diff --git a/source/isaaclab_newton/isaaclab_newton/assets/deformable_object/__init__.pyi b/source/isaaclab_newton/isaaclab_newton/assets/deformable_object/__init__.pyi new file mode 100644 index 00000000000..5878e22680c --- /dev/null +++ b/source/isaaclab_newton/isaaclab_newton/assets/deformable_object/__init__.pyi @@ -0,0 +1,12 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +__all__ = [ + "DeformableObject", + "DeformableObjectData", +] + +from isaaclab_contrib.deformable.deformable_object import DeformableObject +from isaaclab_contrib.deformable.deformable_object_data import DeformableObjectData diff --git a/source/isaaclab_newton/isaaclab_newton/cloner/newton_replicate.py b/source/isaaclab_newton/isaaclab_newton/cloner/newton_replicate.py index e2751e2274f..e99b1eb7abd 100644 --- a/source/isaaclab_newton/isaaclab_newton/cloner/newton_replicate.py +++ b/source/isaaclab_newton/isaaclab_newton/cloner/newton_replicate.py @@ -62,6 +62,25 @@ def _build_newton_builder_from_mapping( # The prototype is built from env_0 in absolute world coordinates. # add_builder xforms are deltas from env_0 so positions don't get double-counted. env0_pos = positions[0] + + # Deformable prim paths are handled by per_world_builder_hooks, not add_usd. + # Resolve the regex prim_path patterns to concrete env_0 paths so add_usd + # can skip them via ignore_paths. + import re + + _deformable_ignore_paths: list[str] = [] + if hasattr(NewtonManager, "_deformable_registry"): + for entry in NewtonManager._deformable_registry: + pat = re.compile(entry.prim_path.replace(".*", "[^/]*") + "$") + for src_path in sources: + # Check if any prim under this source matches the deformable pattern + prim = stage.GetPrimAtPath(src_path) + if prim.IsValid(): + for child in Usd.PrimRange(prim): + child_path = str(child.GetPath()) + if pat.match(child_path): + _deformable_ignore_paths.append(child_path) + protos: dict[str, ModelBuilder] = {} for src_path in sources: p = NewtonManager.create_builder(up_axis=up_axis) @@ -72,6 +91,7 @@ def _build_newton_builder_from_mapping( load_visual_shapes=True, skip_mesh_approximation=True, schema_resolvers=schema_resolvers, + ignore_paths=_deformable_ignore_paths if _deformable_ignore_paths else None, ) if simplify_meshes: p.approximate_meshes("convex_hull", keep_visual_shapes=True) @@ -107,9 +127,20 @@ def _build_newton_builder_from_mapping( local_site_map[label] = [[] for _ in range(num_worlds)] for proto_shape_idx in proto_shape_indices: local_site_map[label][col].append(offset + proto_shape_idx) + + # Run per-world builder hooks (e.g. deformable body registration). + if hasattr(NewtonManager, "_per_world_builder_hooks"): + for hook in NewtonManager._per_world_builder_hooks: + hook(builder, col, positions[col].tolist(), quaternions[col].tolist()) + # end the world context builder.end_world() + # Run post-replicate hooks (e.g. builder.color() for deformable coloring). + if hasattr(NewtonManager, "_post_replicate_hooks"): + for hook in NewtonManager._post_replicate_hooks: + hook(builder) + site_index_map = { **global_site_map, **{label: (None, per_world) for label, per_world in local_site_map.items()}, diff --git a/source/isaaclab_newton/isaaclab_newton/physics/newton_collision_cfg.py b/source/isaaclab_newton/isaaclab_newton/physics/newton_collision_cfg.py index 86014d13a88..9e75d315330 100644 --- a/source/isaaclab_newton/isaaclab_newton/physics/newton_collision_cfg.py +++ b/source/isaaclab_newton/isaaclab_newton/physics/newton_collision_cfg.py @@ -19,9 +19,9 @@ class HydroelasticSDFCfg: Hydroelastic contacts generate distributed contact areas instead of point contacts, providing more realistic force distribution for manipulation and compliant surfaces. - For more details, see the `Newton Collisions Guide`_. + For more details, see the `Newton hydroelastic contacts guide`_. - .. _Newton Collisions Guide: https://newton-physics.github.io/newton/latest/concepts/collisions.html#hydroelastic-contacts + .. _Newton hydroelastic contacts guide: https://newton-physics.github.io/newton/latest/concepts/collisions.html#hydroelastic-contacts """ reduce_contacts: bool = True @@ -88,9 +88,9 @@ class NewtonCollisionPipelineCfg: - Mesh-mesh collision via SDF with contact reduction - Optional hydroelastic contact model for compliant surfaces - For more details, see the `Newton Collisions Guide`_ and `CollisionPipeline API`_. + For more details, see the `Newton collision pipeline guide`_ and `CollisionPipeline API`_. - .. _Newton Collisions Guide: https://newton-physics.github.io/newton/latest/concepts/collisions.html + .. _Newton collision pipeline guide: https://newton-physics.github.io/newton/latest/concepts/collisions.html .. _CollisionPipeline API: https://newton-physics.github.io/newton/api/_generated/newton.CollisionPipeline.html """ diff --git a/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py b/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py index fb8188ff590..f41afa85678 100644 --- a/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py +++ b/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py @@ -33,14 +33,16 @@ from newton.sensors import SensorIMU as NewtonSensorIMU from newton.solvers import SolverBase, SolverKamino, SolverNotifyFlags -from isaaclab.physics import CallbackHandle, PhysicsEvent, PhysicsManager, SceneDataBackend, SceneDataFormat +from isaaclab.physics import CallbackHandle, PhysicsEvent, PhysicsManager +from isaaclab.scene_data import SceneDataBackend, SceneDataFormat +from isaaclab.sim import SimulationContext from isaaclab.sim.utils.newton_model_utils import replace_newton_shape_colors from isaaclab.sim.utils.stage import get_current_stage from isaaclab.utils import checked_apply from isaaclab.utils.string import resolve_matching_names from isaaclab.utils.timer import Timer -from .newton_manager_cfg import NewtonCfg, NewtonShapeCfg, NewtonSolverCfg +from .newton_manager_cfg import NewtonCfg, NewtonShapeCfg if TYPE_CHECKING: from isaaclab.sim.simulation_context import SimulationContext @@ -74,6 +76,35 @@ def _set_fabric_transforms( fabric_transforms[i] = wp.transpose(wp.mat44d(wp.transform_to_matrix(transform))) +@wp.kernel(enable_backward=False) +def _sync_particle_points( + fabric_points: wp.fabricarrayarray(dtype=wp.vec3f), + fabric_world_matrices: wp.fabricarray(dtype=wp.mat44d), + offsets: wp.fabricarray(dtype=wp.uint32), + counts: wp.fabricarray(dtype=wp.uint32), + particle_q: wp.array(dtype=wp.vec3f), +): + """Write Newton particle positions into Fabric mesh point arrays as local-frame points. + + Newton stores particle positions in world space in ``state.particle_q``. The Fabric + ``points`` attribute on a ``UsdGeom.Mesh`` is local-space -- Kit multiplies by the + mesh prim's resolved ``omni:fabric:worldMatrix`` at render time. + + This kernel inverts the mesh prim's world matrix to convert each world-space particle + position into local-space before writing. + """ + i = wp.tid() + offset = int(offsets[i]) + num_points = int(counts[i]) + + # Un-transpose Fabric's stored matrix to get the standard homogeneous form + world_matrix = wp.transpose(wp.mat44f(fabric_world_matrices[i])) + inv_world_matrix = wp.inverse(world_matrix) + + for j in range(num_points): + fabric_points[i][j] = wp.transform_point(inv_world_matrix, particle_q[offset + j]) + + @wp.kernel(enable_backward=False) def _or_reset_masks_from_mask( env_mask: wp.array(dtype=wp.bool), @@ -217,6 +248,9 @@ class NewtonManager(PhysicsManager): _newton_index_attr = "newton:index" _clone_physics_only = False _transforms_dirty: bool = False + _particles_dirty: bool = False + _newton_particle_offset_attr = "newton:particleOffset" + _newton_particle_count_attr = "newton:particleCount" # cubric GPU transform hierarchy (replaces CPU update_world_xforms) _cubric = None @@ -232,8 +266,8 @@ class NewtonManager(PhysicsManager): # Visualization-only state used when the sim backend is PhysX. Populated # lazily in :meth:`_ensure_visualization_model` and updated each render # frame in :meth:`update_visualization_state`. - _visualization_scene_data: SceneDataFormat.Transform | None = None - _visualization_mapping: wp.array | None = None + _scene_data: SceneDataFormat.Transform | None = None + _scene_data_mapping: wp.array | None = None # Views list for assets to register their views _views: list = [] @@ -307,6 +341,7 @@ def forward(cls) -> None: def pre_render(cls) -> None: """Flush deferred Fabric writes before cameras/visualizers read the scene.""" cls.sync_transforms_to_usd() + cls.sync_particles_to_usd() @classmethod def sync_transforms_to_usd(cls) -> None: @@ -406,15 +441,79 @@ def sync_transforms_to_usd(cls) -> None: except Exception: logger.exception("[NewtonManager] sync_transforms_to_usd FAILED") + @classmethod + def sync_particles_to_usd(cls) -> None: + """Write Newton particle_q to Fabric mesh point arrays for Kit viewport rendering. + + For each deformable body whose mesh prim carries a ``newton:particleOffset`` + attribute, this function copies the corresponding slice of ``state_0.particle_q`` + into the Fabric ``points`` array so the Kit viewport reflects the current + deformation. + + No-op when there is no ``_usdrt_stage``, no simulation state, or no + deformable bodies registered. + """ + if cls._usdrt_stage is None or cls._state_0 is None or cls._state_0.particle_q is None: + return + if not cls._particles_dirty: + return + pq = cls._state_0.particle_q + try: + import usdrt + + selection = cls._usdrt_stage.SelectPrims( + require_attrs=[ + (usdrt.Sdf.ValueTypeNames.Point3fArray, "points", usdrt.Usd.Access.ReadWrite), + (usdrt.Sdf.ValueTypeNames.UInt, cls._newton_particle_offset_attr, usdrt.Usd.Access.Read), + (usdrt.Sdf.ValueTypeNames.UInt, cls._newton_particle_count_attr, usdrt.Usd.Access.Read), + (usdrt.Sdf.ValueTypeNames.Matrix4d, "omni:fabric:worldMatrix", usdrt.Usd.Access.Read), + ], + device=str(PhysicsManager._device), + ) + if selection.GetCount() == 0: + return + fabric_points = wp.fabricarrayarray(data=selection, attrib="points", dtype=wp.vec3f) + fabric_offsets = wp.fabricarray(data=selection, attrib=cls._newton_particle_offset_attr) + fabric_counts = wp.fabricarray(data=selection, attrib=cls._newton_particle_count_attr) + fabric_world_matrices = wp.fabricarray(data=selection, attrib="omni:fabric:worldMatrix") + wp.launch( + _sync_particle_points, + dim=selection.GetCount(), + inputs=[fabric_points, fabric_world_matrices, fabric_offsets, fabric_counts, pq], + device=PhysicsManager._device, + ) + NewtonManager._particles_dirty = False + except Exception as exc: + logger.debug("[sync_particles_to_usd] %s", exc) + @classmethod def _mark_transforms_dirty(cls) -> None: - """Flag that physics state has changed and Fabric needs re-sync. + """Flag that rigid-body transforms have changed and Fabric needs re-sync. - Called by :meth:`_simulate` after stepping. The actual sync is deferred - to :meth:`sync_transforms_to_usd`, which runs at render cadence. + The actual sync is deferred to :meth:`sync_transforms_to_usd`, + which runs at render cadence via :meth:`pre_render`. """ NewtonManager._transforms_dirty = True + @classmethod + def _mark_particles_dirty(cls) -> None: + """Flag that particle positions have changed and Fabric needs re-sync. + + The actual sync is deferred to the particle sync callback (if registered), + which runs at render cadence via :meth:`pre_render`. + """ + NewtonManager._particles_dirty = True + + @classmethod + def _mark_state_dirty(cls) -> None: + """Flag that all physics state has changed and Fabric needs re-sync. + + Convenience method that marks both transforms and particles dirty. + Called by :meth:`_simulate` after stepping. + """ + cls._mark_transforms_dirty() + cls._mark_particles_dirty() + @classmethod def step(cls) -> None: """Step the physics simulation. @@ -500,7 +599,7 @@ def step(cls) -> None: PhysicsManager._sim_time += physics_dt if cls._usdrt_stage is not None: - cls._mark_transforms_dirty() + cls._mark_state_dirty() # Launch solver-specific debug logging after stepping. cls._log_solver_debug() @@ -512,7 +611,7 @@ def close(cls) -> None: super().close() @classmethod - def get_scene_data_backend(cls) -> SceneDataBackend: + def get_scene_data_backend(cls) -> SceneDataBackend | None: """Return the SceneDataBackend for the SceneDataProvider.""" return cls._scene_data_backend @@ -547,9 +646,7 @@ def is_fabric_enabled(cls) -> bool: @classmethod def clear(cls): - """Clear all Newton-specific state (callbacks cleared by super().close()). - Start with solver specific cleanup.""" - cls._solver_specific_clear() + """Clear all Newton-specific state (callbacks cleared by super().close()).""" if cls._cubric is not None and cls._cubric_adapter is not None: cls._cubric.release_adapter(cls._cubric_adapter) NewtonManager._cubric = None @@ -586,9 +683,10 @@ def clear(cls): NewtonManager._newton_stage_path = None NewtonManager._usdrt_stage = None NewtonManager._transforms_dirty = False + NewtonManager._particles_dirty = False NewtonManager._up_axis = "Z" - NewtonManager._visualization_scene_data = None - NewtonManager._visualization_mapping = None + NewtonManager._scene_data = None + NewtonManager._scene_data_mapping = None NewtonManager._model_changes = set() NewtonManager._scene_data_backend = None NewtonManager._cl_pending_sites = {} @@ -596,6 +694,7 @@ def clear(cls): NewtonManager._pending_extended_state_attributes = set() NewtonManager._pending_extended_contact_attributes = set() NewtonManager._views = [] + cls._solver_specific_clear() @classmethod def set_builder(cls, builder: ModelBuilder) -> None: @@ -877,6 +976,7 @@ def start_simulation(cls) -> None: if cls._pending_extended_contact_attributes: cls._model.request_contact_attributes(*cls._pending_extended_contact_attributes) NewtonManager._pending_extended_contact_attributes = set() + NewtonManager._state_0 = cls._model.state() NewtonManager._state_1 = cls._model.state() NewtonManager._control = cls._model.control() @@ -902,23 +1002,29 @@ def start_simulation(cls) -> None: import usdrt body_paths = getattr(cls._model, "body_label", None) or getattr(cls._model, "body_key", None) - if body_paths is None: - raise RuntimeError("NewtonManager: model has no body_label/body_key, skipping USD/Fabric sync for RTX.") - NewtonManager._usdrt_stage = get_current_stage(fabric=True) - for i, prim_path in enumerate(body_paths): - prim = cls._usdrt_stage.GetPrimAtPath(prim_path) - prim.CreateAttribute(cls._newton_index_attr, usdrt.Sdf.ValueTypeNames.UInt, True) - prim.GetAttribute(cls._newton_index_attr).Set(i) - # Tag with PhysicsRigidBodyAPI so cubric's eRigidBody mode - # applies Inverse propagation (preserves Newton's world - # transforms and derives local) instead of Forward. - prim.AddAppliedSchema("PhysicsRigidBodyAPI") - xformable_prim = usdrt.Rt.Xformable(prim) - if not xformable_prim.HasWorldXform(): - xformable_prim.SetWorldXformFromUsd() - - cls._mark_transforms_dirty() - cls.sync_transforms_to_usd() + if not body_paths: + logger.warning( + "NewtonManager: model has no rigid bodies (body_label/body_key is empty). " + "USD/Fabric body sync for RTX is skipped. " + "Particle-only scenes (e.g. cloth) must register their own USD mesh update." + ) + NewtonManager._usdrt_stage = None + else: + NewtonManager._usdrt_stage = get_current_stage(fabric=True) + for i, prim_path in enumerate(body_paths): + prim = cls._usdrt_stage.GetPrimAtPath(prim_path) + prim.CreateAttribute(cls._newton_index_attr, usdrt.Sdf.ValueTypeNames.UInt, True) + prim.GetAttribute(cls._newton_index_attr).Set(i) + # Tag with PhysicsRigidBodyAPI so cubric's eRigidBody mode + # applies Inverse propagation (preserves Newton's world + # transforms and derives local) instead of Forward. + prim.AddAppliedSchema("PhysicsRigidBodyAPI") + xformable_prim = usdrt.Rt.Xformable(prim) + if not xformable_prim.HasWorldXform(): + xformable_prim.SetWorldXformFromUsd() + + cls._mark_transforms_dirty() + cls.sync_transforms_to_usd() @classmethod def instantiate_builder_from_stage(cls): @@ -1033,7 +1139,7 @@ def _initialize_contacts(cls) -> None: @classmethod @abstractmethod - def _build_solver(cls, model: Model, solver_cfg: NewtonSolverCfg) -> None: + def _build_solver(cls, model: Model, solver_cfg) -> None: """Construct the solver this manager owns and assign it onto the base class. Subclasses must populate the canonical :class:`NewtonManager` slots: @@ -1062,16 +1168,14 @@ def _build_solver(cls, model: Model, solver_cfg: NewtonSolverCfg) -> None: raise NotImplementedError("NewtonManager subclasses must implement _build_solver()") @classmethod - def _step_solver(cls, state_0: State, state_1: State, control: Control, substep_dt: float) -> None: + def _step_solver( + cls, state_0: State, state_1: State, control: Control, contacts: Contacts | None, substep_dt: float + ) -> None: """Run one solver substep. Default invokes :attr:`_solver` once. Subclasses can override to batch multiple solvers within a single substep. """ - # Only solvers that consume Newton collision-pipeline contacts receive ``_contacts`` here. Solvers with - # internal contact handling receive ``None`` even when ``_contacts`` is allocated for later reporting via - # ``solver.update_contacts`` (e.g. MuJoCo with ``use_mujoco_contacts=True``). - contacts = cls._contacts if cls._needs_collision_pipeline else None cls._solver.step(state_0, state_1, control, contacts, substep_dt) @classmethod @@ -1118,17 +1222,13 @@ def initialize_solver(cls) -> None: NewtonManager._solver_dt = cls.get_physics_dt() / cls._num_substeps NewtonManager._collision_cfg = cfg.collision_cfg # type: ignore[union-attr] - cls._build_solver( - cls._model, - cfg.solver_cfg, # type: ignore[union-attr] - ) + cls._build_solver(cls._model, cfg.solver_cfg) # type: ignore[union-attr] if NewtonManager._solver is None: raise RuntimeError( f"{cls.__name__}._build_solver did not assign NewtonManager._solver. " "Subclasses of NewtonManager must populate NewtonManager._solver, " "NewtonManager._use_single_state, and NewtonManager._needs_collision_pipeline." ) - cls._initialize_contacts() if cls._usdrt_stage is not None: @@ -1342,17 +1442,17 @@ def _run_solver_substeps(cls, contacts) -> None: """Run ``num_substeps`` solver iterations, handling double-buffered state swap.""" if cls._use_single_state: for _ in range(cls._num_substeps): - cls._solver.step(cls._state_0, cls._state_0, cls._control, contacts, cls._solver_dt) + cls._step_solver(cls._state_0, cls._state_0, cls._control, contacts, cls._solver_dt) cls._state_0.clear_forces() else: cfg = PhysicsManager._cfg need_copy_on_last = (cfg is not None and cfg.use_cuda_graph) and cls._num_substeps % 2 == 1 # type: ignore[union-attr] for i in range(cls._num_substeps): - cls._solver.step(cls._state_0, cls._state_1, cls._control, contacts, cls._solver_dt) + cls._step_solver(cls._state_0, cls._state_1, cls._control, contacts, cls._solver_dt) if need_copy_on_last and i == cls._num_substeps - 1: cls._state_0.assign(cls._state_1) else: - cls._state_0, cls._state_1 = cls._state_1, cls._state_0 + NewtonManager._state_0, NewtonManager._state_1 = cls._state_1, cls._state_0 cls._state_0.clear_forces() @classmethod @@ -1413,14 +1513,6 @@ def _simulate_physics_only(cls) -> None: cls._run_solver_substeps(contacts) cls._update_sensors(contacts) - @classmethod - def get_solver_convergence_steps(cls) -> dict[str, float | int]: - """Get the solver convergence steps. Needs to be implemented in solver-specific managers.""" - if hasattr(cls, "_get_solver_convergence_steps"): - return cls._get_solver_convergence_steps() - else: - raise NotImplementedError("NewtonManager subclasses must implement _get_solver_convergence_steps()") - # State accessors (used extensively by articulation/rigid object data) @classmethod def get_model(cls) -> Model: @@ -1442,7 +1534,7 @@ def get_state_0(cls) -> State: return cls._state_0 @classmethod - def get_state(cls) -> State: + def get_state(cls, scene_data_provider=None) -> State: """Get the current Newton state for visualization. Use this method from visualizers/renderers/video recorders that need a @@ -1453,7 +1545,7 @@ def get_state(cls) -> State: :meth:`update_visualization_state` is a no-op and this is equivalent to :meth:`get_state_0`. """ - cls.update_visualization_state() + cls.update_visualization_state(scene_data_provider) return cls.get_state_0() @classmethod @@ -1461,8 +1553,10 @@ def get_num_envs(cls) -> int: return cls._num_envs @classmethod - def _backend_is_newton(cls) -> bool: + def _backend_is_newton(cls, scene_data_provider=None) -> bool: """Return ``True`` when the active sim backend is Newton.""" + if scene_data_provider is not None: + return isinstance(scene_data_provider.backend, NewtonSceneDataBackend) sim = PhysicsManager._sim if sim is None: return False @@ -1481,7 +1575,7 @@ def _ensure_visualization_model(cls) -> None: :meth:`_build_visualization_model_from_stage` and finalizing the resulting :class:`~newton.ModelBuilder`. Per-frame body transforms are pushed into ``_state_0.body_q`` by :meth:`update_visualization_state` using the new - :class:`~isaaclab.scene.scene_data_provider.SceneDataProvider`. + :class:`~isaaclab.scene_data.SceneDataProvider`. """ if cls._model is not None and cls._state_0 is not None: @@ -1519,8 +1613,8 @@ def _ensure_visualization_model(cls) -> None: device = PhysicsManager._device or "cpu" try: - cls._model = builder.finalize(device=device) - cls._state_0 = cls._model.state() + NewtonManager._model = builder.finalize(device=device) + NewtonManager._state_0 = cls._model.state() cls._model.num_envs = cls._num_envs replace_newton_shape_colors(cls._model) @@ -1529,8 +1623,8 @@ def _ensure_visualization_model(cls) -> None: "[NewtonManager] Failed to finalize the shadow Newton ModelBuilder for " "visualization (sim backend is PhysX)." ) - cls._model = None - cls._state_0 = None + NewtonManager._model = None + NewtonManager._state_0 = None @classmethod def _build_visualization_model_from_stage(cls, stage) -> ModelBuilder | None: @@ -1668,14 +1762,14 @@ def _build_visualization_model_from_stage(cls, stage) -> ModelBuilder | None: return builder @classmethod - def update_visualization_state(cls) -> None: + def update_visualization_state(cls, scene_data_provider=None) -> None: """Refresh visualization state for the active sim backend. Newton sim backend: no-op — ``_state_0`` is the live, authoritative state already advanced by :meth:`step` / forward kinematics. PhysX sim backend: pull rigid-body transforms from the - :class:`~isaaclab.scene.scene_data_provider.SceneDataProvider` and write + :class:`~isaaclab.scene_data.SceneDataProvider` and write them into the shadow ``_state_0.body_q`` so Newton-native consumers (Newton renderer, Newton/Rerun/Viser visualizers, OVRTX renderer, Newton GL video) see fresh poses. @@ -1683,28 +1777,27 @@ def update_visualization_state(cls) -> None: Invoked lazily from :meth:`get_state` so consumers do not need to coordinate the sync explicitly. """ - if cls._backend_is_newton(): + if cls._backend_is_newton(scene_data_provider): return cls._ensure_visualization_model() if cls._state_0 is None or cls._model is None or cls._state_0.body_q is None: return - sim = PhysicsManager._sim - if sim is None: + sdp = scene_data_provider + if sdp is None: + sim = SimulationContext.instance() + if sim is not None: + sdp = sim.get_scene_data_provider() + if sdp is None: return - sdp = sim.get_scene_data_provider() - if cls._visualization_scene_data is None: - cls._visualization_scene_data = SceneDataFormat.Transform() - if cls._visualization_mapping is None: + if cls._scene_data is None: + cls._scene_data = SceneDataFormat.Transform() + if cls._scene_data_mapping is None: body_paths = list(getattr(cls._model, "body_label", None) or []) - cls._visualization_mapping = sdp.create_mapping(body_paths) + cls._scene_data_mapping = sdp.create_mapping(body_paths) - cls._visualization_scene_data.transforms = cls._state_0.body_q - sdp.get_transforms( - cls._visualization_scene_data, - mapping=cls._visualization_mapping, - allow_passthrough=False, - ) + cls._scene_data.transforms = cls._state_0.body_q + sdp.get_transforms(cls._scene_data, mapping=cls._scene_data_mapping) @classmethod def get_state_1(cls) -> State: diff --git a/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py b/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py index fe7db9806cf..ac54443971c 100644 --- a/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py +++ b/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py @@ -117,9 +117,8 @@ def update(self, positions: ProxyArray, orientations: ProxyArray, intrinsics: Pr first_focal_length = intrinsics.torch[:, 1, 1][0:1] fov_radians_all = 2.0 * torch.atan(self.height / (2.0 * first_focal_length)) - self.camera_rays = self.newton_sensor.utils.compute_pinhole_camera_rays( - self.width, self.height, wp.from_torch(fov_radians_all, dtype=wp.float32) - ) + fov_warp = wp.from_torch(fov_radians_all, dtype=wp.float32) + self.camera_rays = self.newton_sensor.utils.compute_pinhole_camera_rays(self.width, self.height, fov_warp) @wp.kernel def _update_transforms( diff --git a/source/isaaclab_newton/isaaclab_newton/sim/__init__.pyi b/source/isaaclab_newton/isaaclab_newton/sim/__init__.pyi index aac4c8327cc..699fc15ad4d 100644 --- a/source/isaaclab_newton/isaaclab_newton/sim/__init__.pyi +++ b/source/isaaclab_newton/isaaclab_newton/sim/__init__.pyi @@ -4,7 +4,19 @@ # SPDX-License-Identifier: BSD-3-Clause __all__ = [ + "NewtonDeformableBodyPropertiesCfg", + "NewtonDeformableBodyMaterialCfg", + "NewtonDeformableMaterialCfg", + "NewtonSurfaceDeformableBodyMaterialCfg", + "schemas", + "spawners", "views", ] -from . import views +from . import schemas, spawners, views +from .schemas import NewtonDeformableBodyPropertiesCfg +from .spawners.materials import ( + NewtonDeformableBodyMaterialCfg, + NewtonDeformableMaterialCfg, + NewtonSurfaceDeformableBodyMaterialCfg, +) diff --git a/source/isaaclab_newton/isaaclab_newton/sim/schemas/__init__.py b/source/isaaclab_newton/isaaclab_newton/sim/schemas/__init__.py index 80f943ad46e..e79e72e10df 100644 --- a/source/isaaclab_newton/isaaclab_newton/sim/schemas/__init__.py +++ b/source/isaaclab_newton/isaaclab_newton/sim/schemas/__init__.py @@ -3,15 +3,8 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Newton and MuJoCo simulation schema configuration classes.""" +"""Sub-module containing Newton schema configuration exports.""" -from .schemas_cfg import ( - MujocoJointDrivePropertiesCfg, - MujocoRigidBodyPropertiesCfg, - NewtonArticulationRootPropertiesCfg, - NewtonCollisionPropertiesCfg, - NewtonJointDrivePropertiesCfg, - NewtonMaterialPropertiesCfg, - NewtonMeshCollisionPropertiesCfg, - NewtonRigidBodyPropertiesCfg, -) +from isaaclab.utils.module import lazy_export + +lazy_export() diff --git a/source/isaaclab_newton/isaaclab_newton/sim/schemas/__init__.pyi b/source/isaaclab_newton/isaaclab_newton/sim/schemas/__init__.pyi index 6dae7b8cf2b..1ae2d74c217 100644 --- a/source/isaaclab_newton/isaaclab_newton/sim/schemas/__init__.pyi +++ b/source/isaaclab_newton/isaaclab_newton/sim/schemas/__init__.pyi @@ -3,11 +3,24 @@ # # SPDX-License-Identifier: BSD-3-Clause +__all__ = [ + "MujocoJointDrivePropertiesCfg", + "MujocoRigidBodyPropertiesCfg", + "NewtonArticulationRootPropertiesCfg", + "NewtonCollisionPropertiesCfg", + "NewtonDeformableBodyPropertiesCfg", + "NewtonJointDrivePropertiesCfg", + "NewtonMaterialPropertiesCfg", + "NewtonMeshCollisionPropertiesCfg", + "NewtonRigidBodyPropertiesCfg", +] + from .schemas_cfg import ( MujocoJointDrivePropertiesCfg, MujocoRigidBodyPropertiesCfg, NewtonArticulationRootPropertiesCfg, NewtonCollisionPropertiesCfg, + NewtonDeformableBodyPropertiesCfg, NewtonJointDrivePropertiesCfg, NewtonMaterialPropertiesCfg, NewtonMeshCollisionPropertiesCfg, diff --git a/source/isaaclab_newton/isaaclab_newton/sim/schemas/schemas_cfg.py b/source/isaaclab_newton/isaaclab_newton/sim/schemas/schemas_cfg.py index d98fbc3a236..db039306375 100644 --- a/source/isaaclab_newton/isaaclab_newton/sim/schemas/schemas_cfg.py +++ b/source/isaaclab_newton/isaaclab_newton/sim/schemas/schemas_cfg.py @@ -10,6 +10,7 @@ from isaaclab.sim.schemas.schemas_cfg import ( ArticulationRootBaseCfg, CollisionBaseCfg, + DeformableBodyPropertiesBaseCfg, JointDriveBaseCfg, MeshCollisionBaseCfg, RigidBodyBaseCfg, @@ -39,6 +40,24 @@ class NewtonRigidBodyPropertiesCfg(RigidBodyBaseCfg): _usd_field_exceptions: ClassVar[dict] = {} +@configclass +class NewtonDeformableBodyPropertiesCfg(DeformableBodyPropertiesBaseCfg): + """Newton-specific properties to apply to a deformable body. + + Currently empty. Backend-specific fields can be added here when Newton exposes + a registered deformable body property schema. + + The ``newton:`` namespace is reserved here so future Newton-native + deformable-body fields can be added without an API change. + + See :meth:`~isaaclab.sim.schemas.modify_deformable_body_properties` for more information. + """ + + _usd_namespace: ClassVar[str | None] = "newton" + _usd_applied_schema: ClassVar[str | None] = None + _usd_field_exceptions: ClassVar[dict] = {} + + @configclass class MujocoRigidBodyPropertiesCfg(NewtonRigidBodyPropertiesCfg): """MuJoCo-solver-specific rigid body properties. diff --git a/source/isaaclab_newton/isaaclab_newton/sim/spawners/__init__.py b/source/isaaclab_newton/isaaclab_newton/sim/spawners/__init__.py new file mode 100644 index 00000000000..9e75b244252 --- /dev/null +++ b/source/isaaclab_newton/isaaclab_newton/sim/spawners/__init__.py @@ -0,0 +1,10 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Newton spawner utilities.""" + +from isaaclab.utils.module import lazy_export + +lazy_export() diff --git a/source/isaaclab_newton/isaaclab_newton/sim/spawners/__init__.pyi b/source/isaaclab_newton/isaaclab_newton/sim/spawners/__init__.pyi new file mode 100644 index 00000000000..ad99e610b43 --- /dev/null +++ b/source/isaaclab_newton/isaaclab_newton/sim/spawners/__init__.pyi @@ -0,0 +1,10 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +__all__ = [ + "materials", +] + +from . import materials diff --git a/source/isaaclab_newton/isaaclab_newton/sim/spawners/materials/__init__.py b/source/isaaclab_newton/isaaclab_newton/sim/spawners/materials/__init__.py new file mode 100644 index 00000000000..7ed9dfa18e2 --- /dev/null +++ b/source/isaaclab_newton/isaaclab_newton/sim/spawners/materials/__init__.py @@ -0,0 +1,10 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Sub-module for Newton material configuration exports.""" + +from isaaclab.utils.module import lazy_export + +lazy_export() diff --git a/source/isaaclab_newton/isaaclab_newton/sim/spawners/materials/__init__.pyi b/source/isaaclab_newton/isaaclab_newton/sim/spawners/materials/__init__.pyi new file mode 100644 index 00000000000..c3f13216f78 --- /dev/null +++ b/source/isaaclab_newton/isaaclab_newton/sim/spawners/materials/__init__.pyi @@ -0,0 +1,18 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +__all__ = [ + "spawn_deformable_body_material", + "NewtonDeformableBodyMaterialCfg", + "NewtonDeformableMaterialCfg", + "NewtonSurfaceDeformableBodyMaterialCfg", +] + +from .physics_materials import spawn_deformable_body_material +from .physics_materials_cfg import ( + NewtonDeformableBodyMaterialCfg, + NewtonDeformableMaterialCfg, + NewtonSurfaceDeformableBodyMaterialCfg, +) diff --git a/source/isaaclab_newton/isaaclab_newton/sim/spawners/materials/physics_materials.py b/source/isaaclab_newton/isaaclab_newton/sim/spawners/materials/physics_materials.py new file mode 100644 index 00000000000..1ace5ae3867 --- /dev/null +++ b/source/isaaclab_newton/isaaclab_newton/sim/spawners/materials/physics_materials.py @@ -0,0 +1,10 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Newton-compatible deformable physics material spawning exports.""" + +from isaaclab.sim.spawners.materials.physics_materials import spawn_deformable_body_material + +__all__ = ["spawn_deformable_body_material"] diff --git a/source/isaaclab_newton/isaaclab_newton/sim/spawners/materials/physics_materials_cfg.py b/source/isaaclab_newton/isaaclab_newton/sim/spawners/materials/physics_materials_cfg.py new file mode 100644 index 00000000000..f7b63a8701c --- /dev/null +++ b/source/isaaclab_newton/isaaclab_newton/sim/spawners/materials/physics_materials_cfg.py @@ -0,0 +1,79 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +from __future__ import annotations + +from collections.abc import Callable +from typing import ClassVar + +from isaaclab.sim.spawners.materials.physics_materials_cfg import ( + DeformableBodyMaterialBaseCfg, + SurfaceDeformableBodyMaterialBaseCfg, +) +from isaaclab.utils.configclass import configclass + + +@configclass +class NewtonDeformableMaterialCfg: + """Newton-specific material properties for a deformable body. + + These properties are set with the prefix ``newton:``. + """ + + _usd_namespace: ClassVar[str | None] = "newton" + _usd_applied_schema: ClassVar[str | None] = None + _usd_field_exceptions: ClassVar[dict] = {} + + density: float = 1.0 + """The material density [kg/m^3]. Defaults to 1.0 kg/m^3.""" + + particle_radius: float = 0.008 + """Particle radius [m] used by the Newton backend.""" + + +@configclass +class NewtonDeformableBodyMaterialCfg(DeformableBodyMaterialBaseCfg, NewtonDeformableMaterialCfg): + """Newton-specific physics material parameters for volume deformable bodies.""" + + _usd_namespace: ClassVar[str | None] = "newton" + _usd_applied_schema: ClassVar[str | None] = None + _usd_field_exceptions: ClassVar[dict] = {} + + func: Callable | str = "isaaclab.sim.spawners.materials.physics_materials:spawn_deformable_body_material" + + k_mu: float = 1e5 + """First Lame material parameter [Pa]. Defaults to 1e5 Pa.""" + + k_lambda: float = 1e5 + """Second Lame material parameter [Pa]. Defaults to 1e5 Pa.""" + + k_damp: float = 0.0 + """Damping stiffness for tetrahedral elements [Pa*s]. Defaults to 0.0.""" + + +@configclass +class NewtonSurfaceDeformableBodyMaterialCfg(SurfaceDeformableBodyMaterialBaseCfg, NewtonDeformableMaterialCfg): + """Newton-specific physics material parameters for surface deformable bodies.""" + + _usd_namespace: ClassVar[str | None] = "newton" + _usd_applied_schema: ClassVar[str | None] = None + _usd_field_exceptions: ClassVar[dict] = {} + + func: Callable | str = "isaaclab.sim.spawners.materials.physics_materials:spawn_deformable_body_material" + + tri_ke: float = 1e4 + """Triangle area-preserving stiffness [Pa]. Used by Newton backend for cloth meshes.""" + + tri_ka: float = 1e4 + """Triangle area stiffness [Pa]. Used by Newton backend for cloth meshes.""" + + tri_kd: float = 1.5e-6 + """Triangle area damping [Pa*s]. Used by Newton backend for cloth meshes.""" + + edge_ke: float = 5.0 + """Bending stiffness [N*m]. Used by Newton backend for cloth meshes.""" + + edge_kd: float = 1e-2 + """Bending damping [N*m*s]. Used by Newton backend for cloth meshes.""" diff --git a/source/isaaclab_ovphysx/changelog.d/daniela-move-scene-data.rst b/source/isaaclab_ovphysx/changelog.d/daniela-move-scene-data.rst new file mode 100644 index 00000000000..3a830a73152 --- /dev/null +++ b/source/isaaclab_ovphysx/changelog.d/daniela-move-scene-data.rst @@ -0,0 +1,7 @@ +Changed +^^^^^^^ + +* Updated imports of :class:`~isaaclab.scene_data.SceneDataBackend` and + :class:`~isaaclab.scene_data.SceneDataFormat` in + :mod:`~isaaclab_ovphysx.physics.ovphysx_manager` to their new location in + :mod:`isaaclab.scene_data` (previously :mod:`isaaclab.physics`). diff --git a/source/isaaclab_ovphysx/isaaclab_ovphysx/physics/ovphysx_manager.py b/source/isaaclab_ovphysx/isaaclab_ovphysx/physics/ovphysx_manager.py index 729b06befa9..14eb03ddc5f 100644 --- a/source/isaaclab_ovphysx/isaaclab_ovphysx/physics/ovphysx_manager.py +++ b/source/isaaclab_ovphysx/isaaclab_ovphysx/physics/ovphysx_manager.py @@ -24,7 +24,8 @@ from pxr import UsdPhysics -from isaaclab.physics import PhysicsEvent, PhysicsManager, SceneDataBackend, SceneDataFormat +from isaaclab.physics import PhysicsEvent, PhysicsManager +from isaaclab.scene_data import SceneDataBackend, SceneDataFormat if TYPE_CHECKING: from isaaclab.sim.simulation_context import SimulationContext diff --git a/source/isaaclab_physx/changelog.d/daniela-move-scene-data.rst b/source/isaaclab_physx/changelog.d/daniela-move-scene-data.rst new file mode 100644 index 00000000000..d2e0b7df99a --- /dev/null +++ b/source/isaaclab_physx/changelog.d/daniela-move-scene-data.rst @@ -0,0 +1,6 @@ +Changed +^^^^^^^ + +* Updated imports of :class:`~isaaclab.scene_data.SceneDataBackend` and + :class:`~isaaclab.scene_data.SceneDataFormat` to their new location in + :mod:`isaaclab.scene_data` (previously :mod:`isaaclab.physics`). diff --git a/source/isaaclab_physx/changelog.d/mtrepte-update-debug-viz.skip b/source/isaaclab_physx/changelog.d/mtrepte-update-debug-viz.skip new file mode 100644 index 00000000000..e69de29bb2d diff --git a/source/isaaclab_physx/changelog.d/mym-deformable-backend-split.minor.rst b/source/isaaclab_physx/changelog.d/mym-deformable-backend-split.minor.rst new file mode 100644 index 00000000000..0879ea0a90f --- /dev/null +++ b/source/isaaclab_physx/changelog.d/mym-deformable-backend-split.minor.rst @@ -0,0 +1,11 @@ +Added +^^^^^ + +* Added PhysX-specific deformable property and material cfgs. + +Deprecated +^^^^^^^^^^ + +* Deprecated generic PhysX deformable cfg aliases in favor of + ``PhysxDeformableBodyPropertiesCfg``, ``PhysxDeformableBodyMaterialCfg``, + and ``PhysxSurfaceDeformableBodyMaterialCfg``. diff --git a/source/isaaclab_physx/changelog.d/mym-deformable_experimental.major.rst b/source/isaaclab_physx/changelog.d/mym-deformable_experimental.major.rst new file mode 100644 index 00000000000..aa4a2dad99f --- /dev/null +++ b/source/isaaclab_physx/changelog.d/mym-deformable_experimental.major.rst @@ -0,0 +1,24 @@ +Changed +^^^^^^^ + +* **Breaking:** Moved deformable body schema and material APIs from + :mod:`isaaclab_physx.sim` to :mod:`isaaclab.sim`, and moved deformable object + configuration from :mod:`isaaclab_physx.assets` to :mod:`isaaclab.assets`. + Import :class:`~isaaclab.sim.DeformableBodyPropertiesCfg`, + :func:`~isaaclab.sim.define_deformable_body_properties`, + :func:`~isaaclab.sim.modify_deformable_body_properties`, + :class:`~isaaclab.sim.DeformableObjectSpawnerCfg`, + :class:`~isaaclab.sim.DeformableBodyMaterialCfg`, + :class:`~isaaclab.sim.SurfaceDeformableBodyMaterialCfg`, and + :func:`~isaaclab.sim.spawn_deformable_body_material` from :mod:`isaaclab.sim` + instead of :mod:`isaaclab_physx.sim`; import + :class:`~isaaclab.assets.DeformableObjectCfg` from :mod:`isaaclab.assets` + instead of :mod:`isaaclab_physx.assets`. +* Changed PhysX deformable API documentation to direct users to the + backend-neutral :mod:`isaaclab.assets` and :mod:`isaaclab.sim` imports. + +Fixed +^^^^^ + +* Fixed :class:`~isaaclab_physx.assets.DeformableObject` state writer methods + to accept ``ProxyArray`` inputs without requiring manual conversion. diff --git a/source/isaaclab_physx/docs/CHANGELOG.rst b/source/isaaclab_physx/docs/CHANGELOG.rst index 8cee8088441..416ad468b05 100644 --- a/source/isaaclab_physx/docs/CHANGELOG.rst +++ b/source/isaaclab_physx/docs/CHANGELOG.rst @@ -100,7 +100,7 @@ Removed * **Breaking:** Removed the ``isaaclab_physx.scene_data_providers`` package (``PhysxSceneDataProvider``). The Warp-native - :class:`~isaaclab.scene.scene_data_provider.SceneDataProvider` now exposes + :class:`~isaaclab.scene_data.SceneDataProvider` now exposes PhysX rigid-body transforms via :class:`~isaaclab_physx.physics.PhysxSceneDataBackend`, and the PhysX→Newton state sync used by Newton visualizers/renderers moved to diff --git a/source/isaaclab_physx/isaaclab_physx/assets/deformable_object/__init__.pyi b/source/isaaclab_physx/isaaclab_physx/assets/deformable_object/__init__.pyi index 5ed56fc4f38..8effe50b9d8 100644 --- a/source/isaaclab_physx/isaaclab_physx/assets/deformable_object/__init__.pyi +++ b/source/isaaclab_physx/isaaclab_physx/assets/deformable_object/__init__.pyi @@ -9,6 +9,7 @@ __all__ = [ "DeformableObjectData", ] +from isaaclab.assets.deformable_object import DeformableObjectCfg + from .deformable_object import DeformableObject -from .deformable_object_cfg import DeformableObjectCfg from .deformable_object_data import DeformableObjectData diff --git a/source/isaaclab_physx/isaaclab_physx/assets/deformable_object/deformable_object.py b/source/isaaclab_physx/isaaclab_physx/assets/deformable_object/deformable_object.py index a240cb8203b..759e31d9fb8 100644 --- a/source/isaaclab_physx/isaaclab_physx/assets/deformable_object/deformable_object.py +++ b/source/isaaclab_physx/isaaclab_physx/assets/deformable_object/deformable_object.py @@ -180,7 +180,7 @@ def update(self, dt: float): def write_nodal_state_to_sim_index( self, - nodal_state: torch.Tensor | wp.array, + nodal_state: torch.Tensor | wp.array | ProxyArray, env_ids: Sequence[int] | torch.Tensor | wp.array | None = None, full_data: bool = False, ) -> None: @@ -195,8 +195,10 @@ def write_nodal_state_to_sim_index( env_ids: Environment indices. If None, then all indices are used. full_data: Whether to expect full data. Defaults to False. """ - # Convert warp to torch if needed - if isinstance(nodal_state, wp.array): + # Convert array wrappers to torch for slicing into position and velocity views. + if isinstance(nodal_state, ProxyArray): + nodal_state = nodal_state.torch + elif isinstance(nodal_state, wp.array): nodal_state = wp.to_torch(nodal_state) # set into simulation self.write_nodal_pos_to_sim_index(nodal_state[..., :3], env_ids=env_ids, full_data=full_data) @@ -204,7 +206,7 @@ def write_nodal_state_to_sim_index( def write_nodal_state_to_sim_mask( self, - nodal_state: torch.Tensor | wp.array, + nodal_state: torch.Tensor | wp.array | ProxyArray, env_mask: wp.array | None = None, ) -> None: """Set the nodal state over selected environment mask into the simulation. @@ -225,7 +227,7 @@ def write_nodal_state_to_sim_mask( def write_nodal_pos_to_sim_index( self, - nodal_pos: torch.Tensor | wp.array, + nodal_pos: torch.Tensor | wp.array | ProxyArray, env_ids: Sequence[int] | torch.Tensor | wp.array | None = None, full_data: bool = False, ) -> None: @@ -242,6 +244,8 @@ def write_nodal_pos_to_sim_index( """ # resolve env_ids env_ids = self._resolve_env_ids(env_ids) + if isinstance(nodal_pos, ProxyArray): + nodal_pos = nodal_pos.warp if full_data: self.assert_shape_and_dtype( nodal_pos, (self.num_instances, self.max_sim_vertices_per_body), wp.vec3f, "nodal_pos" @@ -271,7 +275,7 @@ def write_nodal_pos_to_sim_index( def write_nodal_pos_to_sim_mask( self, - nodal_pos: torch.Tensor | wp.array, + nodal_pos: torch.Tensor | wp.array | ProxyArray, env_mask: wp.array | None = None, ) -> None: """Set the nodal positions over selected environment mask into the simulation. @@ -292,7 +296,7 @@ def write_nodal_pos_to_sim_mask( def write_nodal_velocity_to_sim_index( self, - nodal_vel: torch.Tensor | wp.array, + nodal_vel: torch.Tensor | wp.array | ProxyArray, env_ids: Sequence[int] | torch.Tensor | wp.array | None = None, full_data: bool = False, ) -> None: @@ -310,6 +314,8 @@ def write_nodal_velocity_to_sim_index( """ # resolve env_ids env_ids = self._resolve_env_ids(env_ids) + if isinstance(nodal_vel, ProxyArray): + nodal_vel = nodal_vel.warp if full_data: self.assert_shape_and_dtype( nodal_vel, (self.num_instances, self.max_sim_vertices_per_body), wp.vec3f, "nodal_vel" @@ -339,7 +345,7 @@ def write_nodal_velocity_to_sim_index( def write_nodal_velocity_to_sim_mask( self, - nodal_vel: torch.Tensor | wp.array, + nodal_vel: torch.Tensor | wp.array | ProxyArray, env_mask: wp.array | None = None, ) -> None: """Set the nodal velocity over selected environment mask into the simulation. @@ -361,7 +367,7 @@ def write_nodal_velocity_to_sim_mask( def write_nodal_kinematic_target_to_sim_index( self, - targets: torch.Tensor | wp.array, + targets: torch.Tensor | wp.array | ProxyArray, env_ids: Sequence[int] | torch.Tensor | wp.array | None = None, full_data: bool = False, ) -> None: @@ -385,6 +391,8 @@ def write_nodal_kinematic_target_to_sim_index( # resolve env_ids env_ids = self._resolve_env_ids(env_ids) + if isinstance(targets, ProxyArray): + targets = targets.warp if full_data: self.assert_shape_and_dtype( targets, (self.num_instances, self.max_sim_vertices_per_body), wp.vec4f, "targets" @@ -403,7 +411,7 @@ def write_nodal_kinematic_target_to_sim_index( write_nodal_vec4f_to_buffer, dim=(env_ids.shape[0], self.max_sim_vertices_per_body), inputs=[targets, env_ids, full_data], - outputs=[self._data.nodal_kinematic_target], + outputs=[self._data.nodal_kinematic_target.warp], device=self.device, ) # set into simulation @@ -413,7 +421,7 @@ def write_nodal_kinematic_target_to_sim_index( def write_nodal_kinematic_target_to_sim_mask( self, - targets: torch.Tensor | wp.array, + targets: torch.Tensor | wp.array | ProxyArray, env_mask: wp.array | None = None, ) -> None: """Set the kinematic targets of the simulation mesh for the deformable bodies using mask. @@ -442,7 +450,7 @@ def write_nodal_kinematic_target_to_sim_mask( def write_nodal_state_to_sim( self, - nodal_state: torch.Tensor | wp.array, + nodal_state: torch.Tensor | wp.array | ProxyArray, env_ids: Sequence[int] | torch.Tensor | wp.array | None = None, ) -> None: """Deprecated. Please use :meth:`write_nodal_state_to_sim_index` instead.""" @@ -455,7 +463,7 @@ def write_nodal_state_to_sim( def write_nodal_kinematic_target_to_sim( self, - targets: torch.Tensor | wp.array, + targets: torch.Tensor | wp.array | ProxyArray, env_ids: Sequence[int] | torch.Tensor | wp.array | None = None, ) -> None: """Deprecated. Please use :meth:`write_nodal_kinematic_target_to_sim_index` instead.""" @@ -469,7 +477,7 @@ def write_nodal_kinematic_target_to_sim( def write_nodal_pos_to_sim( self, - nodal_pos: torch.Tensor | wp.array, + nodal_pos: torch.Tensor | wp.array | ProxyArray, env_ids: Sequence[int] | torch.Tensor | wp.array | None = None, ) -> None: """Deprecated. Please use :meth:`write_nodal_pos_to_sim_index` instead.""" @@ -482,7 +490,7 @@ def write_nodal_pos_to_sim( def write_nodal_velocity_to_sim( self, - nodal_vel: torch.Tensor | wp.array, + nodal_vel: torch.Tensor | wp.array | ProxyArray, env_ids: Sequence[int] | torch.Tensor | wp.array | None = None, ) -> None: """Deprecated. Please use :meth:`write_nodal_velocity_to_sim_index` instead.""" diff --git a/source/isaaclab_physx/isaaclab_physx/assets/deformable_object/deformable_object_data.py b/source/isaaclab_physx/isaaclab_physx/assets/deformable_object/deformable_object_data.py index 30307c1b08f..df22c8b061b 100644 --- a/source/isaaclab_physx/isaaclab_physx/assets/deformable_object/deformable_object_data.py +++ b/source/isaaclab_physx/isaaclab_physx/assets/deformable_object/deformable_object_data.py @@ -156,7 +156,7 @@ def nodal_state_w(self) -> ProxyArray: wp.launch( compute_nodal_state_w, dim=(self._num_instances, self._max_sim_vertices), - inputs=[self.nodal_pos_w, self.nodal_vel_w], + inputs=[self.nodal_pos_w.warp, self.nodal_vel_w.warp], outputs=[self._nodal_state_w.data], device=self.device, ) @@ -180,7 +180,7 @@ def root_pos_w(self) -> ProxyArray: wp.launch( compute_mean_vec3f_over_vertices, dim=(self._num_instances,), - inputs=[self.nodal_pos_w, self._max_sim_vertices], + inputs=[self.nodal_pos_w.warp, self._max_sim_vertices], outputs=[self._root_pos_w.data], device=self.device, ) @@ -200,7 +200,7 @@ def root_vel_w(self) -> ProxyArray: wp.launch( compute_mean_vec3f_over_vertices, dim=(self._num_instances,), - inputs=[self.nodal_vel_w, self._max_sim_vertices], + inputs=[self.nodal_vel_w.warp, self._max_sim_vertices], outputs=[self._root_vel_w.data], device=self.device, ) diff --git a/source/isaaclab_physx/isaaclab_physx/physics/physx_manager.py b/source/isaaclab_physx/isaaclab_physx/physics/physx_manager.py index bb23bc130ad..df30c9e268a 100644 --- a/source/isaaclab_physx/isaaclab_physx/physics/physx_manager.py +++ b/source/isaaclab_physx/isaaclab_physx/physics/physx_manager.py @@ -32,7 +32,8 @@ from pxr import Sdf, Usd, UsdPhysics, UsdUtils import isaaclab.sim as sim_utils -from isaaclab.physics import CallbackHandle, PhysicsEvent, PhysicsManager, SceneDataBackend, SceneDataFormat +from isaaclab.physics import CallbackHandle, PhysicsEvent, PhysicsManager +from isaaclab.scene_data import SceneDataBackend, SceneDataFormat from isaaclab.utils.string import to_camel_case if TYPE_CHECKING: @@ -335,7 +336,6 @@ def step(cls) -> None: cls._physx_sim.simulate(sim.cfg.dt, 0.0) cls._physx_sim.fetch_results() - device = PhysicsManager._device if "cuda" in device: torch.cuda.set_device(device) @@ -348,6 +348,7 @@ def play(cls) -> None: cls._timeline.play() # Pump events so timeline callbacks fire synchronously omni.kit.app.get_app().update() + cls._sync_fabric_after_resume() @classmethod def pause(cls) -> None: @@ -377,15 +378,20 @@ def wait_for_playing(cls) -> None: app.update() if cls._timeline.is_stopped(): break - # Force fabric to re-sync articulation transforms after resume. - # detach/attach resets the FabricManager, then we immediately push - # current poses so the first render after resume shows correct state. - if not cls._timeline.is_stopped(): - cls._re_sync_fabric() - if cls._view is not None: - cls._view.update_articulations_kinematic() - if cls._update_fabric is not None: - cls._update_fabric(0.0, 0.0) + cls._sync_fabric_after_resume() + + @classmethod + def _sync_fabric_after_resume(cls) -> None: + """Force Fabric to show current articulation transforms after timeline resume.""" + if cls._timeline.is_stopped(): + return + # detach/attach resets the FabricManager, then immediately push current + # poses so the first render after resume shows correct state. + cls._re_sync_fabric() + if cls._view is not None: + cls._view.update_articulations_kinematic() + if cls._update_fabric is not None: + cls._update_fabric(0.0, 0.0) @classmethod def close(cls) -> None: diff --git a/source/isaaclab_physx/isaaclab_physx/sim/__init__.pyi b/source/isaaclab_physx/isaaclab_physx/sim/__init__.pyi index 9a522d96ff8..b3a0bbfcd16 100644 --- a/source/isaaclab_physx/isaaclab_physx/sim/__init__.pyi +++ b/source/isaaclab_physx/isaaclab_physx/sim/__init__.pyi @@ -8,12 +8,14 @@ __all__ = [ "modify_deformable_body_properties", "DeformableBodyPropertiesCfg", "JointDrivePropertiesCfg", + "OmniPhysicsDeformableBodyPropertiesCfg", + "PhysxDeformableBodyPropertiesCfg", "PhysxJointDrivePropertiesCfg", "PhysxRigidBodyPropertiesCfg", "RigidBodyPropertiesCfg", - "DeformableObjectSpawnerCfg", - "spawn_deformable_body_material", "DeformableBodyMaterialCfg", + "PhysxDeformableBodyMaterialCfg", + "PhysxSurfaceDeformableBodyMaterialCfg", "SurfaceDeformableBodyMaterialCfg", "views", ] @@ -23,14 +25,16 @@ from .schemas import ( modify_deformable_body_properties, DeformableBodyPropertiesCfg, JointDrivePropertiesCfg, + OmniPhysicsDeformableBodyPropertiesCfg, + PhysxDeformableBodyPropertiesCfg, PhysxJointDrivePropertiesCfg, PhysxRigidBodyPropertiesCfg, RigidBodyPropertiesCfg, ) from .spawners import ( - DeformableObjectSpawnerCfg, - spawn_deformable_body_material, DeformableBodyMaterialCfg, + PhysxDeformableBodyMaterialCfg, + PhysxSurfaceDeformableBodyMaterialCfg, SurfaceDeformableBodyMaterialCfg, ) from . import views diff --git a/source/isaaclab_physx/isaaclab_physx/sim/schemas/__init__.py b/source/isaaclab_physx/isaaclab_physx/sim/schemas/__init__.py index b0d1477483d..47f0b1e8ae9 100644 --- a/source/isaaclab_physx/isaaclab_physx/sim/schemas/__init__.py +++ b/source/isaaclab_physx/isaaclab_physx/sim/schemas/__init__.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Sub-module containing utilities for schemas used in Omniverse for PhysX backend.""" +"""Sub-module containing PhysX schema configuration exports.""" from isaaclab.utils.module import lazy_export diff --git a/source/isaaclab_physx/isaaclab_physx/sim/schemas/__init__.pyi b/source/isaaclab_physx/isaaclab_physx/sim/schemas/__init__.pyi index 6d2a05bf803..b542edf9f45 100644 --- a/source/isaaclab_physx/isaaclab_physx/sim/schemas/__init__.pyi +++ b/source/isaaclab_physx/isaaclab_physx/sim/schemas/__init__.pyi @@ -14,11 +14,12 @@ __all__ = [ "FixedTendonPropertiesCfg", "JointDrivePropertiesCfg", "MeshCollisionPropertiesCfg", + "OmniPhysicsDeformableBodyPropertiesCfg", "PhysxArticulationRootPropertiesCfg", "PhysxCollisionPropertiesCfg", - "PhysXCollisionPropertiesCfg", "PhysxConvexDecompositionPropertiesCfg", "PhysxConvexHullPropertiesCfg", + "PhysxDeformableBodyPropertiesCfg", "PhysxDeformableCollisionPropertiesCfg", "PhysxFixedTendonPropertiesCfg", "PhysxJointDrivePropertiesCfg", @@ -47,11 +48,12 @@ from .schemas_cfg import ( FixedTendonPropertiesCfg, JointDrivePropertiesCfg, MeshCollisionPropertiesCfg, + OmniPhysicsDeformableBodyPropertiesCfg, PhysxArticulationRootPropertiesCfg, PhysxCollisionPropertiesCfg, - PhysXCollisionPropertiesCfg, PhysxConvexDecompositionPropertiesCfg, PhysxConvexHullPropertiesCfg, + PhysxDeformableBodyPropertiesCfg, PhysxDeformableCollisionPropertiesCfg, PhysxFixedTendonPropertiesCfg, PhysxJointDrivePropertiesCfg, diff --git a/source/isaaclab_physx/isaaclab_physx/sim/schemas/schemas.py b/source/isaaclab_physx/isaaclab_physx/sim/schemas/schemas.py index 5b57b93d7ff..713c1cc3264 100644 --- a/source/isaaclab_physx/isaaclab_physx/sim/schemas/schemas.py +++ b/source/isaaclab_physx/isaaclab_physx/sim/schemas/schemas.py @@ -3,206 +3,12 @@ # # SPDX-License-Identifier: BSD-3-Clause -# needed to import for allowing type-hinting: Usd.Stage | None -from __future__ import annotations - -import logging - -from omni.physx.scripts import deformableUtils -from pxr import Usd - -from isaaclab.sim.utils import ( - apply_nested, - get_all_matching_child_prims, - safe_set_attribute_on_usd_prim, -) -from isaaclab.sim.utils.stage import get_current_stage -from isaaclab.utils.string import to_camel_case - -from isaaclab_physx.sim.schemas.schemas_cfg import DeformableBodyPropertiesCfg - -# import logger -logger = logging.getLogger(__name__) - +"""Compatibility wrappers for deformable schema writers. +The deformable schema writers are backend-aware but remain unified in +:mod:`isaaclab.sim.schemas`. """ -Deformable body properties. -""" - - -def define_deformable_body_properties( - prim_path: str, - cfg: DeformableBodyPropertiesCfg, - stage: Usd.Stage | None = None, - deformable_type: str = "volume", - sim_mesh_prim_path: str | None = None, -): - """Apply the deformable body schema on the input prim and set its properties. - - See :func:`modify_deformable_body_properties` for more details on how the properties are set. - - .. note:: - If the input prim is not a mesh, this function will traverse the prim and find the first mesh - under it. If no mesh or multiple meshes are found, an error is raised. This is because the deformable - body schema can only be applied to a single mesh. - - Args: - prim_path: The prim path where to apply the deformable body schema. - cfg: The configuration for the deformable body. - stage: The stage where to find the prim. Defaults to None, in which case the - current stage is used. - deformable_type: The type of the deformable body (surface or volume). - This is used to determine which PhysX API to use for the deformable body. Defaults to "volume". - sim_mesh_prim_path: Optional override for the simulation mesh prim path. - If None, it is set to ``{prim_path}/sim_mesh`` for surface deformables - and ``{prim_path}/sim_tetmesh`` for volume deformables. - - Raises: - ValueError: When the prim path is not valid. - ValueError: When the prim has no mesh or multiple meshes. - RuntimeError: When setting the deformable body properties fails. - """ - # get stage handle - if stage is None: - stage = get_current_stage() - - # get USD prim - prim = stage.GetPrimAtPath(prim_path) - # check if prim path is valid - if not prim.IsValid(): - raise ValueError(f"Prim path '{prim_path}' is not valid.") - - # traverse the prim and get the mesh. If none or multiple meshes are found, raise error. - matching_prims = get_all_matching_child_prims(prim_path, lambda p: p.GetTypeName() == "Mesh") - # check if the volume deformable mesh is valid - if len(matching_prims) == 0: - raise ValueError(f"Could not find any mesh in '{prim_path}'. Please check asset.") - if len(matching_prims) > 1: - # get list of all meshes found - mesh_paths = [p.GetPrimPath() for p in matching_prims] - raise ValueError( - f"Found multiple meshes in '{prim_path}': {mesh_paths}." - " Deformable body schema can only be applied to one mesh." - ) - mesh_prim = matching_prims[0] - mesh_prim_path = mesh_prim.GetPrimPath() - - # check if the prim is valid - if not mesh_prim.IsValid(): - raise ValueError(f"Mesh prim path '{mesh_prim_path}' is not valid.") - - # set root prim properties based on the type of the deformable mesh (surface vs volume) - if deformable_type == "surface": - sim_mesh_prim_path = prim_path + "/sim_mesh" if sim_mesh_prim_path is None else sim_mesh_prim_path - success = deformableUtils.create_auto_surface_deformable_hierarchy( - stage=stage, - root_prim_path=prim_path, - simulation_mesh_path=sim_mesh_prim_path, - cooking_src_mesh_path=mesh_prim_path, - cooking_src_simplification_enabled=False, - set_visibility_with_guide_purpose=True, - ) - elif deformable_type == "volume": - sim_mesh_prim_path = prim_path + "/sim_tetmesh" if sim_mesh_prim_path is None else sim_mesh_prim_path - success = deformableUtils.create_auto_volume_deformable_hierarchy( - stage=stage, - root_prim_path=prim_path, - simulation_tetmesh_path=sim_mesh_prim_path, - collision_tetmesh_path=sim_mesh_prim_path, - cooking_src_mesh_path=mesh_prim_path, - simulation_hex_mesh_enabled=False, - cooking_src_simplification_enabled=False, - set_visibility_with_guide_purpose=True, - ) - else: - raise ValueError( - f"""Unsupported deformable type: '{deformable_type}'. - Only surface and volume deformables are supported.""" - ) - - # api failure - if not success: - raise RuntimeError(f"Failed to set deformable body properties on prim '{mesh_prim_path}'.") - - # set deformable body properties - modify_deformable_body_properties(prim_path, cfg, stage) - - -@apply_nested -def modify_deformable_body_properties(prim_path: str, cfg: DeformableBodyPropertiesCfg, stage: Usd.Stage | None = None): - """Modify PhysX parameters for a deformable body prim. - - A `deformable body`_ is a single body (either surface or volume deformable) that can be simulated by PhysX. - Unlike rigid bodies, deformable bodies support relative motion of the nodes in the mesh. - Consequently, they can be used to simulate deformations under applied forces. - - PhysX deformable body simulation employs Finite Element Analysis (FEA) to simulate the deformations of the mesh. - It uses two meshes to represent the deformable body: - - 1. **Simulation mesh**: This mesh is used for the simulation and is the one that is deformed by the solver. - 2. **Collision mesh**: This mesh only needs to match the surface of the simulation mesh and is used for - collision detection. - - For most applications, we assume that the above two meshes are computed from the "render mesh" of the deformable - body. The render mesh is the mesh that is visible in the scene and is used for rendering purposes. It is composed - of triangles, while the simulation mesh is composed of tetrahedrons for volume deformables, - and triangles for surface deformables. - - .. caution:: - The deformable body schema is still under development by the Omniverse team. The current implementation - works with the PhysX schemas shipped with Isaac Sim 6.0.0 onwards. It may change in future releases. - - .. note:: - This function is decorated with :func:`apply_nested` that sets the properties to all the prims - (that have the schema applied on them) under the input prim path. - - .. _deformable body: https://nvidia-omniverse.github.io/PhysX/physx/5.6.1/docs/DeformableVolume.html - .. _PhysxDeformableBodyAPI: https://docs.omniverse.nvidia.com/kit/docs/omni_usd_schema_physics/latest/physxschema/annotated.html - - Args: - prim_path: The prim path to the deformable body. - cfg: The configuration for the deformable body. - stage: The stage where to find the prim. Defaults to None, in which case the - current stage is used. - - Returns: - True if the properties were successfully set, False otherwise. - """ - # get stage handle - if stage is None: - stage = get_current_stage() - - # get deformable-body USD prim - deformable_body_prim = stage.GetPrimAtPath(prim_path) - # check if the prim is valid - if not deformable_body_prim.IsValid(): - return False - # check if deformable body API is applied - if "OmniPhysicsDeformableBodyAPI" not in deformable_body_prim.GetAppliedSchemas(): - return False - - # apply customization to deformable API - if "PhysxBaseDeformableBodyAPI" not in deformable_body_prim.GetAppliedSchemas(): - deformable_body_prim.AddAppliedSchema("PhysxBaseDeformableBodyAPI") - # ensure PhysX collision API is applied on the collision mesh - if "PhysxCollisionAPI" not in deformable_body_prim.GetAppliedSchemas(): - deformable_body_prim.AddAppliedSchema("PhysxCollisionAPI") +from isaaclab.sim.schemas.schemas import define_deformable_body_properties, modify_deformable_body_properties - # convert to dict - cfg = cfg.to_dict() - # set into PhysX API - if cfg["kinematic_enabled"]: - logger.warning( - "Kinematic deformable bodies are not fully supported in the current version of Omni Physics. " - "Setting kinematic_enabled to True may lead to unexpected behavior." - ) - # prefixes for each attribute (collision attributes: physxCollision:*, and physxDeformable:* for rest) - property_prefixes = cfg["_property_prefix"] - for prefix, attr_list in property_prefixes.items(): - for attr_name in attr_list: - safe_set_attribute_on_usd_prim( - deformable_body_prim, f"{prefix}:{to_camel_case(attr_name, 'cC')}", cfg[attr_name], camel_case=False - ) - # success - return True +__all__ = ["define_deformable_body_properties", "modify_deformable_body_properties"] diff --git a/source/isaaclab_physx/isaaclab_physx/sim/schemas/schemas_cfg.py b/source/isaaclab_physx/isaaclab_physx/sim/schemas/schemas_cfg.py index 88729c2b0f0..2eacd969e94 100644 --- a/source/isaaclab_physx/isaaclab_physx/sim/schemas/schemas_cfg.py +++ b/source/isaaclab_physx/isaaclab_physx/sim/schemas/schemas_cfg.py @@ -5,13 +5,13 @@ from __future__ import annotations -import dataclasses import warnings from typing import ClassVar from isaaclab.sim.schemas.schemas_cfg import ( ArticulationRootBaseCfg, CollisionBaseCfg, + DeformableBodyPropertiesBaseCfg, JointDriveBaseCfg, MeshCollisionBaseCfg, RigidBodyBaseCfg, @@ -20,15 +20,16 @@ @configclass -class OmniPhysicsPropertiesCfg: +class OmniPhysicsDeformableBodyPropertiesCfg(DeformableBodyPropertiesBaseCfg): """OmniPhysics properties for a deformable body. - These properties are set with the prefix ``omniphysics:``. For example, to set the mass of the - deformable body, you would set the property ``omniphysics:mass``. - - See the OmniPhysics documentation for more information on the available properties. + These properties are set with the prefix ``omniphysics:``. """ + _usd_namespace: ClassVar[str | None] = "omniphysics" + _usd_applied_schema: ClassVar[str | None] = None + _usd_field_exceptions: ClassVar[dict] = {} + deformable_body_enabled: bool | None = None """Enables deformable body.""" @@ -36,7 +37,7 @@ class OmniPhysicsPropertiesCfg: """Enables kinematic body. Defaults to False, which means that the body is not kinematic.""" mass: float | None = None - """The material mass in [kg]. Defaults to None, in which case the material density is used to compute the mass.""" + """The material mass [kg]. Defaults to None, in which case the material density is used to compute the mass.""" @configclass @@ -48,6 +49,10 @@ class PhysXDeformableBodyPropertiesCfg: For more information on the available properties, please refer to the `documentation `_. """ + _usd_namespace: ClassVar[str | None] = "physxDeformableBody" + _usd_applied_schema: ClassVar[str | None] = "PhysxBaseDeformableBodyAPI" + _usd_field_exceptions: ClassVar[dict] = {} + solver_position_iteration_count: int = 16 """Number of the solver positional iterations per step. Range is [1,255], default to 16.""" @@ -134,6 +139,10 @@ class PhysxDeformableCollisionPropertiesCfg: as a base of :class:`DeformableBodyPropertiesCfg`. """ + _usd_namespace: ClassVar[str | None] = "physxCollision" + _usd_applied_schema: ClassVar[str | None] = "PhysxCollisionAPI" + _usd_field_exceptions: ClassVar[dict] = {} + contact_offset: float | None = None """Contact offset for the collision shape [m]. @@ -152,31 +161,12 @@ class PhysxDeformableCollisionPropertiesCfg: @configclass -class PhysXCollisionPropertiesCfg(PhysxDeformableCollisionPropertiesCfg): - """Deprecated: use :class:`PhysxDeformableCollisionPropertiesCfg`. - - .. deprecated:: 4.6.23 - ``PhysXCollisionPropertiesCfg`` (capital X) was renamed to - :class:`PhysxDeformableCollisionPropertiesCfg` to clear the namespace for the - new rigid-body :class:`PhysxCollisionPropertiesCfg` (lowercase x). The capital-X - name is preserved as a deprecation alias and is scheduled for removal in 5.0. - """ - - def __post_init__(self): - warnings.warn( - "'PhysXCollisionPropertiesCfg' (capital X) is deprecated and will be removed in 5.0." - " Use 'isaaclab_physx.sim.schemas.PhysxDeformableCollisionPropertiesCfg' instead.", - DeprecationWarning, - stacklevel=2, - ) - super().__post_init__() - - -@configclass -class DeformableBodyPropertiesCfg( - OmniPhysicsPropertiesCfg, PhysXDeformableBodyPropertiesCfg, PhysxDeformableCollisionPropertiesCfg +class PhysxDeformableBodyPropertiesCfg( + OmniPhysicsDeformableBodyPropertiesCfg, + PhysXDeformableBodyPropertiesCfg, + PhysxDeformableCollisionPropertiesCfg, ): - """Properties to apply to a deformable body. + """PhysX-specific properties to apply to a deformable body. A deformable body is a body that can deform under forces, both surface and volume deformables. The configuration allows users to specify the properties of the deformable body, @@ -192,12 +182,25 @@ class DeformableBodyPropertiesCfg( the properties and leave the rest as-is. """ - _property_prefix: dict[str, list[str]] = { - "omniphysics": [field.name for field in dataclasses.fields(OmniPhysicsPropertiesCfg)], - "physxDeformableBody": [field.name for field in dataclasses.fields(PhysXDeformableBodyPropertiesCfg)], - "physxCollision": [field.name for field in dataclasses.fields(PhysxDeformableCollisionPropertiesCfg)], - } - """Mapping between the property prefixes and the properties that fall under each prefix.""" + +@configclass +class DeformableBodyPropertiesCfg(PhysxDeformableBodyPropertiesCfg): + """Deprecated: use :class:`PhysxDeformableBodyPropertiesCfg`. + + .. deprecated:: 4.6.x + ``DeformableBodyPropertiesCfg`` has moved to + :class:`PhysxDeformableBodyPropertiesCfg` for PhysX-specific deformable properties + and is scheduled for removal in 5.0. + """ + + def __post_init__(self): + warnings.warn( + "'DeformableBodyPropertiesCfg' is deprecated and will be removed in 5.0. Use" + " 'isaaclab_physx.sim.schemas.PhysxDeformableBodyPropertiesCfg' instead.", + DeprecationWarning, + stacklevel=2, + ) + super().__post_init__() @configclass diff --git a/source/isaaclab_physx/isaaclab_physx/sim/spawners/__init__.py b/source/isaaclab_physx/isaaclab_physx/sim/spawners/__init__.py index 82b931452f4..767a7adeb7e 100644 --- a/source/isaaclab_physx/isaaclab_physx/sim/spawners/__init__.py +++ b/source/isaaclab_physx/isaaclab_physx/sim/spawners/__init__.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Sub-module containing utilities for creating prims in Omniverse for the PhysX backend.""" +"""Sub-module containing PhysX spawner compatibility exports.""" from isaaclab.utils.module import lazy_export diff --git a/source/isaaclab_physx/isaaclab_physx/sim/spawners/__init__.pyi b/source/isaaclab_physx/isaaclab_physx/sim/spawners/__init__.pyi index 805a9f79fd5..617e4566c1d 100644 --- a/source/isaaclab_physx/isaaclab_physx/sim/spawners/__init__.pyi +++ b/source/isaaclab_physx/isaaclab_physx/sim/spawners/__init__.pyi @@ -4,15 +4,15 @@ # SPDX-License-Identifier: BSD-3-Clause __all__ = [ - "DeformableObjectSpawnerCfg", - "spawn_deformable_body_material", "DeformableBodyMaterialCfg", + "PhysxDeformableBodyMaterialCfg", + "PhysxSurfaceDeformableBodyMaterialCfg", "SurfaceDeformableBodyMaterialCfg", ] -from .spawner_cfg import DeformableObjectSpawnerCfg from .materials import ( - spawn_deformable_body_material, DeformableBodyMaterialCfg, + PhysxDeformableBodyMaterialCfg, + PhysxSurfaceDeformableBodyMaterialCfg, SurfaceDeformableBodyMaterialCfg, ) diff --git a/source/isaaclab_physx/isaaclab_physx/sim/spawners/materials/__init__.py b/source/isaaclab_physx/isaaclab_physx/sim/spawners/materials/__init__.py index 4877bb0c147..8ef9db1d4d8 100644 --- a/source/isaaclab_physx/isaaclab_physx/sim/spawners/materials/__init__.py +++ b/source/isaaclab_physx/isaaclab_physx/sim/spawners/materials/__init__.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Sub-module for spawners that spawn PhysX-based materials.""" +"""Sub-module for PhysX material configuration compatibility exports.""" from isaaclab.utils.module import lazy_export diff --git a/source/isaaclab_physx/isaaclab_physx/sim/spawners/materials/__init__.pyi b/source/isaaclab_physx/isaaclab_physx/sim/spawners/materials/__init__.pyi index 1a3e833d61d..315458ea5d2 100644 --- a/source/isaaclab_physx/isaaclab_physx/sim/spawners/materials/__init__.pyi +++ b/source/isaaclab_physx/isaaclab_physx/sim/spawners/materials/__init__.pyi @@ -4,17 +4,21 @@ # SPDX-License-Identifier: BSD-3-Clause __all__ = [ - "spawn_deformable_body_material", "DeformableBodyMaterialCfg", + "PhysXDeformableMaterialCfg", + "PhysxDeformableBodyMaterialCfg", "PhysxRigidBodyMaterialCfg", + "PhysxSurfaceDeformableBodyMaterialCfg", "RigidBodyMaterialCfg", "SurfaceDeformableBodyMaterialCfg", ] -from .physics_materials import spawn_deformable_body_material from .physics_materials_cfg import ( DeformableBodyMaterialCfg, + PhysXDeformableMaterialCfg, + PhysxDeformableBodyMaterialCfg, PhysxRigidBodyMaterialCfg, + PhysxSurfaceDeformableBodyMaterialCfg, RigidBodyMaterialCfg, SurfaceDeformableBodyMaterialCfg, ) diff --git a/source/isaaclab_physx/isaaclab_physx/sim/spawners/materials/physics_materials.py b/source/isaaclab_physx/isaaclab_physx/sim/spawners/materials/physics_materials.py deleted file mode 100644 index 78920cca9f8..00000000000 --- a/source/isaaclab_physx/isaaclab_physx/sim/spawners/materials/physics_materials.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -from pxr import Usd, UsdShade - -from isaaclab.sim.utils import clone, safe_set_attribute_on_usd_prim -from isaaclab.sim.utils.stage import get_current_stage -from isaaclab.utils.string import to_camel_case - -from . import physics_materials_cfg - - -@clone -def spawn_deformable_body_material(prim_path: str, cfg: physics_materials_cfg.DeformableBodyMaterialCfg) -> Usd.Prim: - """Create material with deformable-body physics properties. - - Deformable body materials are used to define the physical properties to meshes of a deformable body. These - include the friction and deformable body properties. For more information on deformable body material, - please refer to the documentation on `PxFEMSoftBodyMaterial`_. - - .. note:: - This function is decorated with :func:`clone` that resolves prim path into list of paths - if the input prim path is a regex pattern. This is done to support spawning multiple assets - from a single and cloning the USD prim at the given path expression. - - Args: - prim_path: The prim path or pattern to spawn the asset at. If the prim path is a regex pattern, - then the asset is spawned at all the matching prim paths. - cfg: The configuration for the physics material. - - Returns: - The spawned deformable body material prim. - - Raises: - ValueError: When a prim already exists at the specified prim path and is not a material. - - .. _PxFEMSoftBodyMaterial: https://nvidia-omniverse.github.io/PhysX/physx/5.4.1/_api_build/structPxFEMSoftBodyMaterialModel.html - """ - # get stage handle - stage = get_current_stage() - - # create material prim if no prim exists - if not stage.GetPrimAtPath(prim_path).IsValid(): - _ = UsdShade.Material.Define(stage, prim_path) - - # obtain prim - prim = stage.GetPrimAtPath(prim_path) - # check if prim is a material - if not prim.IsA(UsdShade.Material): - raise ValueError(f"A prim already exists at path: '{prim_path}' but is not a material.") - # ensure PhysX deformable body material API is applied - applied = prim.GetAppliedSchemas() - if "OmniPhysicsDeformableMaterialAPI" not in applied: - prim.AddAppliedSchema("OmniPhysicsDeformableMaterialAPI") - if "PhysxDeformableMaterialAPI" not in applied: - prim.AddAppliedSchema("PhysxDeformableMaterialAPI") - # surface deformable material API - is_surface_deformable = isinstance(cfg, physics_materials_cfg.SurfaceDeformableBodyMaterialCfg) - if is_surface_deformable: - if "OmniPhysicsSurfaceDeformableMaterialAPI" not in applied: - prim.AddAppliedSchema("OmniPhysicsSurfaceDeformableMaterialAPI") - if "PhysxSurfaceDeformableMaterialAPI" not in applied: - prim.AddAppliedSchema("PhysxSurfaceDeformableMaterialAPI") - - # convert to dict - cfg = cfg.to_dict() - del cfg["func"] - # set into PhysX API, gather prefixes for each attribute - property_prefixes = cfg["_property_prefix"] - for prefix, attr_list in property_prefixes.items(): - for attr_name in attr_list: - safe_set_attribute_on_usd_prim( - prim, f"{prefix}:{to_camel_case(attr_name, 'cC')}", cfg[attr_name], camel_case=False - ) - # return the prim - return prim diff --git a/source/isaaclab_physx/isaaclab_physx/sim/spawners/materials/physics_materials_cfg.py b/source/isaaclab_physx/isaaclab_physx/sim/spawners/materials/physics_materials_cfg.py index 7128a080b50..345fadc8870 100644 --- a/source/isaaclab_physx/isaaclab_physx/sim/spawners/materials/physics_materials_cfg.py +++ b/source/isaaclab_physx/isaaclab_physx/sim/spawners/materials/physics_materials_cfg.py @@ -5,13 +5,15 @@ from __future__ import annotations -import dataclasses import warnings from collections.abc import Callable from typing import ClassVar, Literal -from isaaclab.sim.spawners.materials import PhysicsMaterialCfg -from isaaclab.sim.spawners.materials.physics_materials_cfg import RigidBodyMaterialBaseCfg +from isaaclab.sim.spawners.materials.physics_materials_cfg import ( + DeformableBodyMaterialBaseCfg, + RigidBodyMaterialBaseCfg, + SurfaceDeformableBodyMaterialBaseCfg, +) from isaaclab.utils.configclass import configclass @@ -19,50 +21,39 @@ class OmniPhysicsDeformableMaterialCfg: """OmniPhysics material properties for a deformable body. - These properties are set with the prefix ``omniphysics:``. For example, to set the density of the - deformable body, you would set the property ``omniphysics:density``. - - See the OmniPhysics documentation for more information on the available properties. + These properties are set with the prefix ``omniphysics:``. """ - density: float | None = None - """The material density in [kg/m^3]. Defaults to None, in which case the simulation decides the default density.""" + _usd_namespace: ClassVar[str | None] = "omniphysics" + _usd_applied_schema: ClassVar[str | None] = "OmniPhysicsDeformableMaterialAPI" + _usd_field_exceptions: ClassVar[dict] = {} + + density: float = 1000.0 + """The material density [kg/m^3]. Defaults to 1000.0 kg/m^3.""" static_friction: float = 0.25 - """The static friction. Defaults to 0.25.""" + """The static friction coefficient. Defaults to 0.25.""" dynamic_friction: float = 0.25 - """The dynamic friction. Defaults to 0.25.""" + """The dynamic friction coefficient. Defaults to 0.25.""" youngs_modulus: float = 1000000.0 - """The Young's modulus, which defines the body's stiffness. Defaults to 1[MPa]. - - The Young's modulus is a measure of the material's ability to deform under stress. It is measured in Pascals ([Pa]). - """ + """The Young's modulus, which defines the body's stiffness [Pa]. Defaults to 1 MPa.""" poissons_ratio: float = 0.45 - """The Poisson's ratio which defines the body's volume preservation. Defaults to 0.45. - - The Poisson's ratio is a measure of the material's ability to expand in the lateral direction when compressed - in the axial direction. It is a dimensionless number between 0 and 0.5. Using a value of 0.5 will make the - material incompressible. - """ + """The Poisson's ratio which defines the body's volume preservation.""" @configclass class OmniPhysicsSurfaceDeformableMaterialCfg(OmniPhysicsDeformableMaterialCfg): - """OmniPhysics material properties for a surface deformable body, - extending on :class:`OmniPhysicsDeformableMaterialCfg` with additional parameters for surface deformable bodies. + """OmniPhysics material properties for a surface deformable body.""" - These properties are set with the prefix ``omniphysics:``. - For example, to set the surface thickness of the surface deformable body, - you would set the property ``omniphysics:surfaceThickness``. - - See the OmniPhysics documentation for more information on the available properties. - """ + _usd_namespace: ClassVar[str | None] = "omniphysics" + _usd_applied_schema: ClassVar[str | None] = "OmniPhysicsSurfaceDeformableMaterialAPI" + _usd_field_exceptions: ClassVar[dict] = {} surface_thickness: float = 0.01 - """The thickness of the deformable body's surface. Defaults to 0.01 meters ([m]).""" + """The thickness of the deformable body's surface [m]. Defaults to 0.01.""" surface_stretch_stiffness: float = 0.0 """The stretch stiffness of the deformable body's surface. Defaults to 0.0.""" @@ -73,56 +64,88 @@ class OmniPhysicsSurfaceDeformableMaterialCfg(OmniPhysicsDeformableMaterialCfg): surface_bend_stiffness: float = 0.0 """The bend stiffness of the deformable body's surface. Defaults to 0.0.""" - bend_damping: float = 0.0 - """The bend damping for the deformable body's surface. Defaults to 0.0.""" - @configclass class PhysXDeformableMaterialCfg: """PhysX-specific material properties for a deformable body. - These properties are set with the prefix ``physxDeformableBody:``. - For example, to set the elasticity damping of the deformable body, - you would set the property ``physxDeformableBody:elasticityDamping``. - - See the PhysX documentation for more information on the available properties. + These properties are set with the prefix ``physxDeformableMaterial:``. """ + _usd_namespace: ClassVar[str | None] = "physxDeformableMaterial" + _usd_applied_schema: ClassVar[str | None] = "PhysxDeformableMaterialAPI" + _usd_field_exceptions: ClassVar[dict] = {} + elasticity_damping: float = 0.005 """The elasticity damping for the deformable material. Defaults to 0.005.""" @configclass -class DeformableBodyMaterialCfg(PhysicsMaterialCfg, OmniPhysicsDeformableMaterialCfg, PhysXDeformableMaterialCfg): - """Physics material parameters for deformable bodies. +class PhysxDeformableBodyMaterialCfg( + DeformableBodyMaterialBaseCfg, + OmniPhysicsDeformableMaterialCfg, + PhysXDeformableMaterialCfg, +): + """PhysX-specific physics material parameters for deformable bodies.""" - See :meth:`spawn_deformable_body_material` for more information. - """ + func: Callable | str = "isaaclab.sim.spawners.materials.physics_materials:spawn_deformable_body_material" + + +@configclass +class PhysxSurfaceDeformableBodyMaterialCfg( + SurfaceDeformableBodyMaterialBaseCfg, + OmniPhysicsSurfaceDeformableMaterialCfg, + PhysXDeformableMaterialCfg, +): + """PhysX-specific physics material parameters for surface deformable bodies.""" + + _usd_namespace: ClassVar[str | None] = "physxDeformableMaterial" + _usd_applied_schema: ClassVar[str | None] = "PhysxSurfaceDeformableMaterialAPI" - func: Callable | str = "{DIR}.physics_materials:spawn_deformable_body_material" + func: Callable | str = "isaaclab.sim.spawners.materials.physics_materials:spawn_deformable_body_material" - _property_prefix: dict[str, list[str]] = { - "omniphysics": [field.name for field in dataclasses.fields(OmniPhysicsDeformableMaterialCfg)], - "physxDeformableBody": [field.name for field in dataclasses.fields(PhysXDeformableMaterialCfg)], - } - """Mapping between the property prefixes and the properties that fall under each prefix.""" + bend_damping: float = 0.0 + """Damping acting against bend-resistance forces [1/s]. Defaults to 0.0.""" @configclass -class SurfaceDeformableBodyMaterialCfg(DeformableBodyMaterialCfg, OmniPhysicsSurfaceDeformableMaterialCfg): - """Physics material parameters for surface deformable bodies, - extending on :class:`DeformableBodyMaterialCfg` with additional parameters for surface deformable bodies. +class DeformableBodyMaterialCfg(PhysxDeformableBodyMaterialCfg): + """Deprecated: use :class:`PhysxDeformableBodyMaterialCfg`. - See :meth:`spawn_deformable_body_material` for more information. + .. deprecated:: 4.6.x + ``DeformableBodyMaterialCfg`` has moved to + :class:`PhysxDeformableBodyMaterialCfg` for PhysX-specific deformable materials + and is scheduled for removal in 5.0. """ - func: Callable | str = "{DIR}.physics_materials:spawn_deformable_body_material" + def __post_init__(self): + warnings.warn( + "'DeformableBodyMaterialCfg' is deprecated and will be removed in 5.0. Use" + " 'isaaclab_physx.sim.spawners.materials.PhysxDeformableBodyMaterialCfg' instead.", + DeprecationWarning, + stacklevel=2, + ) + super().__post_init__() + - _property_prefix: dict[str, list[str]] = { - "omniphysics": [field.name for field in dataclasses.fields(OmniPhysicsSurfaceDeformableMaterialCfg)], - "physxDeformableBody": [field.name for field in dataclasses.fields(PhysXDeformableMaterialCfg)], - } - """Extend DeformableBodyMaterialCfg properties under each prefix.""" +@configclass +class SurfaceDeformableBodyMaterialCfg(PhysxSurfaceDeformableBodyMaterialCfg): + """Deprecated: use :class:`PhysxSurfaceDeformableBodyMaterialCfg`. + + .. deprecated:: 4.6.x + ``SurfaceDeformableBodyMaterialCfg`` has moved to + :class:`PhysxSurfaceDeformableBodyMaterialCfg` for PhysX-specific surface + deformable materials and is scheduled for removal in 5.0. + """ + + def __post_init__(self): + warnings.warn( + "'SurfaceDeformableBodyMaterialCfg' is deprecated and will be removed in 5.0. Use" + " 'isaaclab_physx.sim.spawners.materials.PhysxSurfaceDeformableBodyMaterialCfg' instead.", + DeprecationWarning, + stacklevel=2, + ) + super().__post_init__() @configclass diff --git a/source/isaaclab_physx/isaaclab_physx/sim/spawners/spawner_cfg.py b/source/isaaclab_physx/isaaclab_physx/sim/spawners/spawner_cfg.py deleted file mode 100644 index d4843ec83d3..00000000000 --- a/source/isaaclab_physx/isaaclab_physx/sim/spawners/spawner_cfg.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -from typing import TYPE_CHECKING - -from isaaclab.sim.spawners.spawner_cfg import SpawnerCfg -from isaaclab.utils.configclass import configclass - -if TYPE_CHECKING: - from isaaclab.sim import schemas - - # deformables only supported on PhysX backend - from isaaclab_physx.sim.schemas.schemas_cfg import DeformableBodyPropertiesCfg - - -@configclass -class DeformableObjectSpawnerCfg(SpawnerCfg): - """Configuration parameters for spawning a deformable asset. - - Unlike rigid objects, deformable objects are affected by forces and can deform when subjected to - external forces. This class is used to configure the properties of the deformable object. - - Deformable bodies don't have a separate collision mesh. The collision mesh is the same as the visual mesh. - The collision properties such as rest and collision offsets are specified in the :attr:`deformable_props`. - - Note: - By default, all properties are set to None. This means that no properties will be added or modified - to the prim outside of the properties available by default when spawning the prim. - """ - - mass_props: schemas.MassPropertiesCfg | None = None - """Mass properties.""" - - deformable_props: DeformableBodyPropertiesCfg | None = None - """Deformable body properties. Only supported on PhysX backend for now.""" diff --git a/source/isaaclab_physx/test/assets/test_deformable_object.py b/source/isaaclab_physx/test/assets/test_deformable_object.py index 31a199938c9..7a884fe4e0e 100644 --- a/source/isaaclab_physx/test/assets/test_deformable_object.py +++ b/source/isaaclab_physx/test/assets/test_deformable_object.py @@ -22,13 +22,18 @@ import torch import warp as wp from flaky import flaky -from isaaclab_physx.assets import DeformableObject, DeformableObjectCfg -from isaaclab_physx.sim import DeformableBodyMaterialCfg, DeformableBodyPropertiesCfg, SurfaceDeformableBodyMaterialCfg +from isaaclab_physx.assets import DeformableObject +from isaaclab_physx.sim import ( + PhysxDeformableBodyMaterialCfg, + PhysxDeformableBodyPropertiesCfg, + PhysxSurfaceDeformableBodyMaterialCfg, +) import carb import isaaclab.sim as sim_utils import isaaclab.utils.math as math_utils +from isaaclab.assets import DeformableObjectCfg from isaaclab.sim import build_simulation_context # Temporarily disabled: this suite intermittently aborts with SIGABRT on CI. @@ -72,14 +77,14 @@ def generate_cubes_scene( if has_api: spawn_cfg = sim_utils.MeshCuboidCfg( size=(0.2, 0.2, 0.2), - deformable_props=DeformableBodyPropertiesCfg(kinematic_enabled=kinematic_enabled), + deformable_props=PhysxDeformableBodyPropertiesCfg(kinematic_enabled=kinematic_enabled), ) # Add physics material if provided if material_path is not None: if deformable_type == "surface": - spawn_cfg.physics_material = SurfaceDeformableBodyMaterialCfg() + spawn_cfg.physics_material = PhysxSurfaceDeformableBodyMaterialCfg() else: - spawn_cfg.physics_material = DeformableBodyMaterialCfg() + spawn_cfg.physics_material = PhysxDeformableBodyMaterialCfg() spawn_cfg.physics_material_path = material_path else: spawn_cfg.physics_material = None @@ -108,8 +113,15 @@ def sim(): yield sim -@pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("material_path", [None, "/World/SoftMaterial", "material"]) +@pytest.mark.parametrize( + "num_cubes, material_path", + [ + (1, "material"), + (2, None), + (2, "/World/SoftMaterial"), + (2, "material"), + ], +) def test_initialization(sim, num_cubes, material_path): """Test initialization for prim with deformable body API at the provided prim path.""" cube_object = generate_cubes_scene(num_cubes=num_cubes, material_path=material_path) @@ -153,10 +165,10 @@ def test_initialization(sim, num_cubes, material_path): assert cube_object.data.root_vel_w.torch.shape == (num_cubes, 3) -@pytest.mark.parametrize("num_cubes", [1, 2]) @pytest.mark.isaacsim_ci -def test_initialization_surface_deformable(sim, num_cubes): +def test_initialization_surface_deformable(sim): """Test initialization of a surface deformable body.""" + num_cubes = 2 cube_object = generate_cubes_scene(num_cubes=num_cubes, deformable_type="surface") # Play sim @@ -203,10 +215,10 @@ def test_initialization_on_device_cpu(): sim.reset() -@pytest.mark.parametrize("num_cubes", [1, 2]) @pytest.mark.isaacsim_ci -def test_set_nodal_state(sim, num_cubes): +def test_set_nodal_state(sim): """Test setting the state of the deformable object.""" + num_cubes = 2 cube_object = generate_cubes_scene(num_cubes=num_cubes) # Play the simulator @@ -241,9 +253,15 @@ def test_set_nodal_state(sim, num_cubes): cube_object.update(sim.cfg.dt) -@pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("randomize_pos", [True, False]) -@pytest.mark.parametrize("randomize_rot", [True, False]) +@pytest.mark.parametrize( + "num_cubes, randomize_pos, randomize_rot", + [ + (1, False, False), + (1, True, False), + (1, False, True), + (2, True, True), + ], +) @flaky(max_runs=3, min_passes=1) @pytest.mark.isaacsim_ci def test_set_nodal_state_with_applied_transform(num_cubes, randomize_pos, randomize_rot): @@ -289,10 +307,10 @@ def test_set_nodal_state_with_applied_transform(num_cubes, randomize_pos, random torch.testing.assert_close(cube_object.data.root_pos_w.torch, mean_nodal_pos_init, rtol=1e-4, atol=1e-4) -@pytest.mark.parametrize("num_cubes", [2, 4]) @pytest.mark.isaacsim_ci -def test_set_kinematic_targets(sim, num_cubes): +def test_set_kinematic_targets(sim): """Test setting kinematic targets for the deformable object.""" + num_cubes = 2 cube_object = generate_cubes_scene(num_cubes=num_cubes, height=1.0) sim.reset() diff --git a/source/isaaclab_physx/test/sim/test_spawn_materials.py b/source/isaaclab_physx/test/sim/test_spawn_materials.py index 477132c5c5a..bdd39a2d242 100644 --- a/source/isaaclab_physx/test/sim/test_spawn_materials.py +++ b/source/isaaclab_physx/test/sim/test_spawn_materials.py @@ -14,7 +14,10 @@ import pytest -from isaaclab_physx.sim.spawners.materials.physics_materials_cfg import DeformableBodyMaterialCfg +from isaaclab_physx.sim.spawners.materials.physics_materials_cfg import ( + PhysxDeformableBodyMaterialCfg, + PhysxSurfaceDeformableBodyMaterialCfg, +) import isaaclab.sim as sim_utils from isaaclab.sim import SimulationCfg, SimulationContext @@ -36,7 +39,7 @@ def sim(): def test_spawn_deformable_body_material(sim): """Test spawning a deformable body material.""" - cfg = DeformableBodyMaterialCfg( + cfg = PhysxDeformableBodyMaterialCfg( density=1.0, dynamic_friction=0.25, youngs_modulus=50000000.0, @@ -52,4 +55,23 @@ def test_spawn_deformable_body_material(sim): assert prim.GetAttribute("omniphysics:dynamicFriction").Get() == cfg.dynamic_friction assert prim.GetAttribute("omniphysics:youngsModulus").Get() == cfg.youngs_modulus assert prim.GetAttribute("omniphysics:poissonsRatio").Get() == cfg.poissons_ratio - assert prim.GetAttribute("physxDeformableBody:elasticityDamping").Get() == pytest.approx(cfg.elasticity_damping) + assert prim.GetAttribute("physxDeformableMaterial:elasticityDamping").Get() == pytest.approx(cfg.elasticity_damping) + + +def test_spawn_surface_deformable_body_material(sim): + """Test spawning a surface deformable body material.""" + cfg = PhysxSurfaceDeformableBodyMaterialCfg( + density=1.0, + youngs_modulus=50000000.0, + poissons_ratio=0.5, + elasticity_damping=0.005, + bend_damping=0.01, + ) + prim = cfg.func("/Looks/SurfaceDeformableBodyMaterial", cfg) + # Check validity + assert prim.IsValid() + assert sim.stage.GetPrimAtPath("/Looks/SurfaceDeformableBodyMaterial").IsValid() + # Check PhysX properties + assert "PhysxSurfaceDeformableMaterialAPI" in prim.GetAppliedSchemas() + assert prim.GetAttribute("physxDeformableMaterial:elasticityDamping").Get() == pytest.approx(cfg.elasticity_damping) + assert prim.GetAttribute("physxDeformableMaterial:bendDamping").Get() == pytest.approx(cfg.bend_damping) diff --git a/source/isaaclab_physx/test/sim/test_spawn_meshes.py b/source/isaaclab_physx/test/sim/test_spawn_meshes.py index 7c24e289ef9..02a21641071 100644 --- a/source/isaaclab_physx/test/sim/test_spawn_meshes.py +++ b/source/isaaclab_physx/test/sim/test_spawn_meshes.py @@ -14,8 +14,8 @@ import pytest -from isaaclab_physx.sim.schemas.schemas_cfg import DeformableBodyPropertiesCfg -from isaaclab_physx.sim.spawners.materials.physics_materials_cfg import DeformableBodyMaterialCfg +from isaaclab_physx.sim.schemas.schemas_cfg import PhysxDeformableBodyPropertiesCfg +from isaaclab_physx.sim.spawners.materials.physics_materials_cfg import PhysxDeformableBodyMaterialCfg import isaaclab.sim as sim_utils from isaaclab.sim import SimulationCfg, SimulationContext @@ -46,33 +46,13 @@ def sim(): """ -def test_spawn_cone_with_deformable_props(sim): - """Test spawning of UsdGeomMesh prim for a cone with deformable body API.""" - # Spawn cone - cfg = sim_utils.MeshConeCfg( - radius=1.0, - height=2.0, - deformable_props=DeformableBodyPropertiesCfg(deformable_body_enabled=True), - ) - prim = cfg.func("/World/Cone", cfg) - - # Check validity - assert prim.IsValid() - assert sim.stage.GetPrimAtPath("/World/Cone").IsValid() - - # Check properties - # Unlike rigid body, deformable body properties are on the mesh prim - prim = sim.stage.GetPrimAtPath("/World/Cone") - assert prim.GetAttribute("omniphysics:deformableBodyEnabled").Get() == cfg.deformable_props.deformable_body_enabled - - def test_spawn_cone_with_deformable_and_mass_props(sim): """Test spawning of UsdGeomMesh prim for a cone with deformable body and mass API.""" # Spawn cone cfg = sim_utils.MeshConeCfg( radius=1.0, height=2.0, - deformable_props=DeformableBodyPropertiesCfg(deformable_body_enabled=True, mass=1.0), + deformable_props=PhysxDeformableBodyPropertiesCfg(deformable_body_enabled=True, mass=1.0), ) prim = cfg.func("/World/Cone", cfg) @@ -81,6 +61,7 @@ def test_spawn_cone_with_deformable_and_mass_props(sim): assert sim.stage.GetPrimAtPath("/World/Cone").IsValid() # Check properties prim = sim.stage.GetPrimAtPath("/World/Cone") + assert prim.GetAttribute("omniphysics:deformableBodyEnabled").Get() == cfg.deformable_props.deformable_body_enabled assert prim.GetAttribute("omniphysics:mass").Get() == cfg.deformable_props.mass # check sim playing @@ -97,8 +78,8 @@ def test_spawn_cone_with_deformable_and_density_props(sim): cfg = sim_utils.MeshConeCfg( radius=1.0, height=2.0, - deformable_props=DeformableBodyPropertiesCfg(deformable_body_enabled=True), - physics_material=DeformableBodyMaterialCfg(density=10.0), + deformable_props=PhysxDeformableBodyPropertiesCfg(deformable_body_enabled=True), + physics_material=PhysxDeformableBodyMaterialCfg(density=10.0), ) prim = cfg.func("/World/Cone", cfg) @@ -113,30 +94,3 @@ def test_spawn_cone_with_deformable_and_density_props(sim): sim.play() for _ in range(10): sim.step() - - -def test_spawn_cone_with_all_deformable_props(sim): - """Test spawning of UsdGeomMesh prim for a cone with all deformable properties.""" - # Spawn cone - cfg = sim_utils.MeshConeCfg( - radius=1.0, - height=2.0, - deformable_props=DeformableBodyPropertiesCfg(mass=1.0), - visual_material=sim_utils.PreviewSurfaceCfg(diffuse_color=(0.0, 0.75, 0.5)), - physics_material=DeformableBodyMaterialCfg(), - ) - prim = cfg.func("/World/Cone", cfg) - - # Check validity - assert prim.IsValid() - assert sim.stage.GetPrimAtPath("/World/Cone").IsValid() - assert sim.stage.GetPrimAtPath("/World/Cone/geometry/material").IsValid() - # Check properties - # -- deformable body - prim = sim.stage.GetPrimAtPath("/World/Cone") - assert prim.GetAttribute("omniphysics:deformableBodyEnabled").Get() is True - - # check sim playing - sim.play() - for _ in range(10): - sim.step() diff --git a/source/isaaclab_rl/changelog.d/mtrepte-update-debug-viz.skip b/source/isaaclab_rl/changelog.d/mtrepte-update-debug-viz.skip new file mode 100644 index 00000000000..e69de29bb2d diff --git a/source/isaaclab_tasks/changelog.d/mtrepte-update-debug-viz.skip b/source/isaaclab_tasks/changelog.d/mtrepte-update-debug-viz.skip new file mode 100644 index 00000000000..e69de29bb2d diff --git a/source/isaaclab_tasks/changelog.d/mym-deformable-backend-split.rst b/source/isaaclab_tasks/changelog.d/mym-deformable-backend-split.rst new file mode 100644 index 00000000000..e0cd3070591 --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/mym-deformable-backend-split.rst @@ -0,0 +1,6 @@ +Changed +^^^^^^^ + +* Changed Franka soft-object task configs to use backend-specific deformable cfgs. + Use Newton deformable cfgs from :mod:`isaaclab_newton.sim` or PhysX deformable + cfgs from :mod:`isaaclab_physx.sim` when customizing these tasks. diff --git a/source/isaaclab_tasks/changelog.d/mym-deformable_experimental.minor.rst b/source/isaaclab_tasks/changelog.d/mym-deformable_experimental.minor.rst new file mode 100644 index 00000000000..561a08c3e26 --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/mym-deformable_experimental.minor.rst @@ -0,0 +1,6 @@ +Added +^^^^^ + +* Added manager-based Franka soft-body lifting environment + ``Isaac-Lift-Soft-Franka-v0`` as the documented rigid-deformable coupling + task. diff --git a/source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/lift/config/franka/ik_abs_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/lift/config/franka/ik_abs_env_cfg.py index 388837be412..e9a151a89a2 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/lift/config/franka/ik_abs_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/lift/config/franka/ik_abs_env_cfg.py @@ -3,8 +3,10 @@ # # SPDX-License-Identifier: BSD-3-Clause -from isaaclab_physx.assets import DeformableObjectCfg +from isaaclab_physx.sim.schemas import PhysxDeformableBodyPropertiesCfg +from isaaclab_physx.sim.spawners.materials import PhysxDeformableBodyMaterialCfg +from isaaclab.assets import DeformableObjectCfg from isaaclab.controllers.differential_ik_cfg import DifferentialIKControllerCfg from isaaclab.envs.mdp.actions.actions_cfg import DifferentialInverseKinematicsActionCfg from isaaclab.managers import EventTermCfg as EventTerm @@ -77,6 +79,8 @@ def __post_init__(self): spawn=UsdFileCfg( usd_path=f"{ISAACLAB_NUCLEUS_DIR}/Objects/Teddy_Bear/teddy_bear.usd", scale=(0.01, 0.01, 0.01), + deformable_props=PhysxDeformableBodyPropertiesCfg(), + physics_material=PhysxDeformableBodyMaterialCfg(), ), ) diff --git a/source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/lift_franka_soft/__init__.py b/source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/lift_franka_soft/__init__.py new file mode 100644 index 00000000000..0a4c52549e0 --- /dev/null +++ b/source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/lift_franka_soft/__init__.py @@ -0,0 +1,33 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +import gymnasium as gym + +from . import agents + +## +# Register Gym environments. +## + + +gym.register( + id="Isaac-Lift-Soft-Franka-v0", + entry_point="isaaclab.envs:ManagerBasedRLEnv", + disable_env_checker=True, + kwargs={ + "env_cfg_entry_point": f"{__name__}.franka_soft_env_cfg:FrankaSoftEnvCfg", + "rsl_rl_cfg_entry_point": f"{agents.__name__}.rsl_rl_ppo_cfg:FrankaDeformablePPORunnerCfg", + }, +) + +gym.register( + id="Isaac-Lift-Cloth-Franka-v0", + entry_point="isaaclab.envs:ManagerBasedRLEnv", + disable_env_checker=True, + kwargs={ + "env_cfg_entry_point": f"{__name__}.franka_cloth_env_cfg:FrankaClothEnvCfg", + "rsl_rl_cfg_entry_point": f"{agents.__name__}.rsl_rl_ppo_cfg:FrankaDeformablePPORunnerCfg", + }, +) diff --git a/source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/lift_franka_soft/agents/__init__.py b/source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/lift_franka_soft/agents/__init__.py new file mode 100644 index 00000000000..460a3056908 --- /dev/null +++ b/source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/lift_franka_soft/agents/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause diff --git a/source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/lift_franka_soft/agents/rsl_rl_ppo_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/lift_franka_soft/agents/rsl_rl_ppo_cfg.py new file mode 100644 index 00000000000..dfbad6c6fb8 --- /dev/null +++ b/source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/lift_franka_soft/agents/rsl_rl_ppo_cfg.py @@ -0,0 +1,38 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +from isaaclab.utils.configclass import configclass + +from isaaclab_rl.rsl_rl import RslRlOnPolicyRunnerCfg, RslRlPpoActorCriticCfg, RslRlPpoAlgorithmCfg + + +@configclass +class FrankaDeformablePPORunnerCfg(RslRlOnPolicyRunnerCfg): + num_steps_per_env = 24 + max_iterations = 50000 + save_interval = 50 + experiment_name = "franka_deformable" + policy = RslRlPpoActorCriticCfg( + init_noise_std=1.0, + actor_obs_normalization=False, + critic_obs_normalization=False, + actor_hidden_dims=[256, 128, 64], + critic_hidden_dims=[256, 128, 64], + activation="elu", + ) + algorithm = RslRlPpoAlgorithmCfg( + value_loss_coef=1.0, + use_clipped_value_loss=True, + clip_param=0.2, + entropy_coef=0.006, + num_learning_epochs=5, + num_mini_batches=4, + learning_rate=1.0e-4, + schedule="adaptive", + gamma=0.98, + lam=0.95, + desired_kl=0.01, + max_grad_norm=1.0, + ) diff --git a/source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/lift_franka_soft/franka_cloth_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/lift_franka_soft/franka_cloth_env_cfg.py new file mode 100644 index 00000000000..1f89cf012c2 --- /dev/null +++ b/source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/lift_franka_soft/franka_cloth_env_cfg.py @@ -0,0 +1,242 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Configuration for the Franka surface deformable lifting environment.""" + +from __future__ import annotations + +from isaaclab_newton.physics import MJWarpSolverCfg +from isaaclab_newton.sim.schemas import NewtonDeformableBodyPropertiesCfg +from isaaclab_newton.sim.spawners.materials import NewtonSurfaceDeformableBodyMaterialCfg + +import isaaclab.sim as sim_utils +from isaaclab.assets import AssetBaseCfg +from isaaclab.assets.deformable_object import DeformableObjectCfg +from isaaclab.managers import EventTermCfg as EventTerm +from isaaclab.managers import RewardTermCfg as RewTerm +from isaaclab.managers import SceneEntityCfg +from isaaclab.utils.configclass import configclass + +from isaaclab_contrib.deformable.newton_manager_cfg import CoupledMJWarpVBDSolverCfg, NewtonModelCfg, VBDSolverCfg + +from isaaclab_tasks.utils import PresetCfg + +from . import mdp +from .franka_soft_env_cfg import DeformableNewtonCfg, FrankaSoftEnvCfg, _FrankaSoftSceneCfg +from .franka_soft_env_cfg import EventCfg as FrankaSoftEventCfg + +## +# Scene definition +## + +ROBOT_SHAPE_MATERIAL_MU = 100.0 +"""Franka collision-shape friction coefficient [dimensionless] used for Newton cloth contact.""" + +ROBOT_SHAPE_MATERIAL_BODY_NAMES = ".*" +"""Franka body-name regex receiving :data:`ROBOT_SHAPE_MATERIAL_MU`.""" + + +@configclass +class PhysicsCfg(PresetCfg): + # Newton physics: MJWarp rigid + VBD soft, one-way coupled + # (matches newton/examples/softbody/example_softbody_franka.py) + newton_mjwarp_vdb: DeformableNewtonCfg = DeformableNewtonCfg( + solver_cfg=CoupledMJWarpVBDSolverCfg( + rigid_solver_cfg=MJWarpSolverCfg( + njmax=40, + nconmax=20, + ls_iterations=20, + cone="pyramidal", + impratio=1, + ls_parallel=False, + integrator="implicitfast", + ccd_iterations=100, + ), + soft_solver_cfg=VBDSolverCfg( + iterations=10, + integrate_with_external_rigid_solver=True, + particle_enable_self_contact=False, + particle_collision_detection_interval=-1, + ), + coupling_mode="two_way", + ), + model_cfg=NewtonModelCfg( + soft_contact_ke=1e3, + soft_contact_kd=1e-5, + soft_contact_mu=0.5, + shape_material_ke=1e3, + shape_material_kd=1e-5, + shape_material_mu=1e-4, + ), + num_substeps=10, + use_cuda_graph=True, + ) + + default = newton_mjwarp_vdb + + +@configclass +class DeformableCfg(PresetCfg): + """Preset config for the deformable object, matching the Newton example.""" + + newton_mjwarp_vdb: DeformableObjectCfg = DeformableObjectCfg( + prim_path="/World/envs/env_.*/Deformable", + init_state=DeformableObjectCfg.InitialStateCfg(pos=(0.4, 0.0, 0.2)), + spawn=sim_utils.MeshRectangleCfg( + size=(0.2, 0.2), + resolution=(30, 30), + deformable_props=NewtonDeformableBodyPropertiesCfg(), + visual_material=sim_utils.PreviewSurfaceCfg(diffuse_color=(0.95, 0.85, 0.1)), + physics_material=NewtonSurfaceDeformableBodyMaterialCfg( + density=50.0, + particle_radius=0.005, + tri_ke=5e2, + tri_ka=5e2, + tri_kd=1e-3, + edge_ke=2.0, + edge_kd=1e-3, + ), + ), + ) + + default = newton_mjwarp_vdb + + +@configclass +class FrankaClothSceneCfg(_FrankaSoftSceneCfg): + """Scene for the Franka surface deformable environment.""" + + deformable: DeformableCfg = DeformableCfg() + + # static collidable cubes the cloth drops onto (sits on the table top at z = 0). + # Modeled as a static asset (no rigid body / no DOFs) so adding it does not + # extend the Newton model's joint state. + cube: AssetBaseCfg = AssetBaseCfg( + prim_path="{ENV_REGEX_NS}/Cube", + init_state=AssetBaseCfg.InitialStateCfg(pos=(0.45, 0.0, 0.04)), + spawn=sim_utils.CuboidCfg( + size=(0.03, 0.01, 0.08), + collision_props=sim_utils.CollisionPropertiesCfg(), + visual_material=sim_utils.PreviewSurfaceCfg(diffuse_color=(0.2, 0.2, 0.25)), + ), + ) + + +@configclass +class ActionsCfg: + """7-dim arm joint position + 1-dim binary gripper.""" + + arm_action = mdp.JointPositionActionCfg( + asset_name="robot", joint_names=["panda_joint.*"], scale=0.1, use_default_offset=True + ) + gripper_action = mdp.BinaryJointPositionActionCfg( + asset_name="robot", + joint_names=["panda_finger.*"], + open_command_expr={"panda_finger_.*": 0.05}, + close_command_expr={"panda_finger_.*": 0.0}, + ) + + +@configclass +class RewardsCfg: + """Lift-to-target reward for a deformable object.""" + + reaching_deformable = RewTerm( + func=mdp.deformable_ee_distance, + params={"std": 0.1, "asset_cfg": SceneEntityCfg("deformable")}, + weight=5.0, + ) + lifting_deformable = RewTerm( + func=mdp.deformable_lifted, + params={"minimal_height": 0.04, "asset_cfg": SceneEntityCfg("deformable")}, + weight=5.0, + ) + deformable_goal_tracking = RewTerm( + func=mdp.deformable_com_goal_distance, + params={ + "std": 0.3, + "minimal_height": 0.075, + "command_name": "deformable_pose", + "asset_cfg": SceneEntityCfg("deformable"), + }, + weight=16.0, + ) + deformable_goal_tracking_fine_grained = RewTerm( + func=mdp.deformable_com_goal_distance, + params={ + "std": 0.05, + "minimal_height": 0.075, + "command_name": "deformable_pose", + "asset_cfg": SceneEntityCfg("deformable"), + }, + weight=5.0, + ) + + action_rate = RewTerm(func=mdp.action_rate_l2, weight=-1e-2) + gripper_close = RewTerm( + func=mdp.gripper_close_action, + params={"action_name": "gripper_action"}, + weight=-1.0, + ) + joint_vel = RewTerm(func=mdp.joint_vel_l2, weight=-1e-2) + joint_torque = RewTerm(func=mdp.joint_torques_l2, weight=-1e-4) + joint_acc = RewTerm(func=mdp.joint_acc_l2, weight=-1e-4) + + +@configclass +class EventCfg(FrankaSoftEventCfg): + """Reset and startup events for the Franka cloth environment.""" + + robot_physics_material = EventTerm( + func=mdp.randomize_rigid_body_material, + mode="startup", + params={ + "asset_cfg": SceneEntityCfg("robot", body_names=ROBOT_SHAPE_MATERIAL_BODY_NAMES), + "static_friction_range": (ROBOT_SHAPE_MATERIAL_MU, ROBOT_SHAPE_MATERIAL_MU), + "dynamic_friction_range": (ROBOT_SHAPE_MATERIAL_MU, ROBOT_SHAPE_MATERIAL_MU), + "restitution_range": (0.0, 0.0), + "num_buckets": 1, + }, + ) + + +## +# Environment configuration +## + + +@configclass +class FrankaClothEnvCfg(FrankaSoftEnvCfg): + """Manager-based RL environment: Franka Panda lifting a surface deformable.""" + + # Scene settings + scene: FrankaClothSceneCfg = FrankaClothSceneCfg(num_envs=128, env_spacing=2.5, replicate_physics=True) + # Basic settings + actions: ActionsCfg = ActionsCfg() + # MDP settings + rewards: RewardsCfg = RewardsCfg() + events: EventCfg = EventCfg() + + def __post_init__(self) -> None: + # general settings + self.decimation = 1 + self.episode_length_s = 5.0 + + # simulation settings + self.sim.dt = 1 / 60.0 + self.sim.render_interval = self.decimation + + # viewer settings + self.viewer.origin_type = "asset_root" + self.viewer.asset_name = "robot" + self.viewer.env_index = 0 + self.viewer.eye = (1.25, -1.5, 0.6) + self.viewer.resolution = (1920, 1080) + self.sim.physics = PhysicsCfg() + + # increase franka gripper stiffness + self.scene.robot.actuators["panda_hand"].effort_limit_sim = 500.0 + self.scene.robot.actuators["panda_hand"].stiffness = 2000.0 + self.scene.robot.actuators["panda_hand"].damping = 100.0 diff --git a/source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/lift_franka_soft/franka_soft_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/lift_franka_soft/franka_soft_env_cfg.py new file mode 100644 index 00000000000..9aba49d303c --- /dev/null +++ b/source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/lift_franka_soft/franka_soft_env_cfg.py @@ -0,0 +1,448 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Configuration for the Franka deformable lifting environment. + +The scene mirrors ``newton/examples/softbody/example_softbody_franka.py``: +a Franka Panda manipulator on a tabletop with a tetrahedral deformable object simulated +by VBD. The RL task is to lift the deformable object's centre of mass to a randomised target +position sampled in the robot's root frame. +""" + +from __future__ import annotations + +from isaaclab_newton.physics import MJWarpSolverCfg, NewtonCfg +from isaaclab_newton.sim.schemas import NewtonDeformableBodyPropertiesCfg +from isaaclab_newton.sim.spawners.materials import NewtonDeformableBodyMaterialCfg +from isaaclab_physx.physics import PhysxCfg +from isaaclab_physx.sim.schemas import PhysxDeformableBodyPropertiesCfg +from isaaclab_physx.sim.spawners.materials import PhysxDeformableBodyMaterialCfg + +import isaaclab.sim as sim_utils +from isaaclab.assets import ArticulationCfg, AssetBaseCfg +from isaaclab.assets.deformable_object import DeformableObjectCfg +from isaaclab.controllers.differential_ik_cfg import DifferentialIKControllerCfg +from isaaclab.envs import ManagerBasedRLEnvCfg +from isaaclab.envs.mdp.actions.actions_cfg import DifferentialInverseKinematicsActionCfg +from isaaclab.managers import EventTermCfg as EventTerm +from isaaclab.managers import ObservationGroupCfg as ObsGroup +from isaaclab.managers import ObservationTermCfg as ObsTerm +from isaaclab.managers import RewardTermCfg as RewTerm +from isaaclab.managers import SceneEntityCfg +from isaaclab.managers import TerminationTermCfg as DoneTerm +from isaaclab.markers import VisualizationMarkersCfg +from isaaclab.scene import InteractiveSceneCfg +from isaaclab.sensors import FrameTransformerCfg +from isaaclab.sensors.frame_transformer.frame_transformer_cfg import OffsetCfg +from isaaclab.sim.spawners.from_files.from_files_cfg import GroundPlaneCfg, UsdFileCfg +from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR +from isaaclab.utils.configclass import configclass + +from isaaclab_contrib.deformable.newton_manager_cfg import CoupledMJWarpVBDSolverCfg, NewtonModelCfg, VBDSolverCfg + +from isaaclab_tasks.utils import PresetCfg + +from . import mdp + +## +# Pre-defined configs +## + +from isaaclab_assets.robots.franka import FRANKA_PANDA_CFG # isort:skip + + +## +# Helpers +## + + +# Shared volume material parameters. The Newton config below uses the equivalent Lame parameters. +YOUNGS_MODULUS = 8e4 +POISSONS_RATIO = 0.25 + + +@configclass +class DeformableNewtonCfg(NewtonCfg): + """NewtonCfg extended with model-level contact parameters for deformable objects. + + Uses a distinct class name so that ``_is_kitless_physics`` does not + match it, ensuring Kit is launched for USD deformable spawning. + """ + + model_cfg: NewtonModelCfg | None = None + """Global Newton model parameters applied after builder finalization.""" + + +@configclass +class DeformableCfg(PresetCfg): + """Preset config for the deformable object, matching the Newton example.""" + + newton_mjwarp_vbd: DeformableObjectCfg = DeformableObjectCfg( + prim_path="/World/envs/env_.*/Deformable", + init_state=DeformableObjectCfg.InitialStateCfg(pos=(0.5, 0.0, 0.05)), + spawn=sim_utils.MeshCuboidCfg( + size=(0.3, 0.05, 0.05), + deformable_props=NewtonDeformableBodyPropertiesCfg(), + visual_material=sim_utils.PreviewSurfaceCfg(diffuse_color=(0.95, 0.85, 0.1)), + physics_material=NewtonDeformableBodyMaterialCfg( + density=300.0, + k_mu=YOUNGS_MODULUS / (2.0 * (1.0 + POISSONS_RATIO)), + k_lambda=(YOUNGS_MODULUS * POISSONS_RATIO / ((1.0 + POISSONS_RATIO) * (1.0 - 2.0 * POISSONS_RATIO))), + particle_radius=0.01, + ), + ), + ) + + physx: DeformableObjectCfg = DeformableObjectCfg( + prim_path="/World/envs/env_.*/Deformable", + init_state=DeformableObjectCfg.InitialStateCfg(pos=(0.5, 0.0, 0.05)), + spawn=sim_utils.MeshCuboidCfg( + size=(0.3, 0.05, 0.05), + deformable_props=PhysxDeformableBodyPropertiesCfg(), + visual_material=sim_utils.PreviewSurfaceCfg(diffuse_color=(0.95, 0.85, 0.1)), + physics_material=PhysxDeformableBodyMaterialCfg( + density=300.0, + youngs_modulus=YOUNGS_MODULUS, + poissons_ratio=POISSONS_RATIO, + static_friction=10.0, + dynamic_friction=5.0, + ), + ), + ) + + default = newton_mjwarp_vbd + + +@configclass +class PhysicsCfg(PresetCfg): + # Newton physics: MJWarp rigid + VBD soft, one-way coupled + # (matches newton/examples/softbody/example_softbody_franka.py) + newton_mjwarp_vbd: DeformableNewtonCfg = DeformableNewtonCfg( + solver_cfg=CoupledMJWarpVBDSolverCfg( + rigid_solver_cfg=MJWarpSolverCfg( + njmax=40, + nconmax=20, + ls_iterations=20, + cone="pyramidal", + impratio=1, + ls_parallel=False, + integrator="implicitfast", + ccd_iterations=100, + ), + soft_solver_cfg=VBDSolverCfg( + iterations=10, + integrate_with_external_rigid_solver=True, + particle_enable_self_contact=False, + particle_collision_detection_interval=-1, + ), + coupling_mode="two_way", + ), + model_cfg=NewtonModelCfg( + soft_contact_ke=1e4, + soft_contact_kd=1e-5, + soft_contact_mu=5.0, + shape_material_ke=4e4, + shape_material_kd=1e-5, + shape_material_mu=5.0, + ), + num_substeps=10, + use_cuda_graph=True, + ) + + physx: PhysxCfg = PhysxCfg() + + default = newton_mjwarp_vbd + + +## +# Scene definition +## + + +@configclass +class _FrankaSoftSceneCfg(InteractiveSceneCfg): + """Scene for the Franka deformable environment.""" + + robot: ArticulationCfg = FRANKA_PANDA_CFG.replace(prim_path="/World/envs/env_.*/Robot") + + # end-effector frame for reward shaping + ee_frame: FrameTransformerCfg = FrameTransformerCfg( + prim_path="/World/envs/env_.*/Robot/panda_link0", + debug_vis=False, + target_frames=[ + FrameTransformerCfg.FrameCfg( + prim_path="/World/envs/env_.*/Robot/panda_hand", + name="end_effector", + offset=OffsetCfg(pos=[0.0, 0.0, 0.1034]), + ), + ], + ) + + deformable: DeformableCfg = DeformableCfg() + + # static table matching the Newton example: half-extents (0.4, 0.4, 0.1) → top at z = 0.2 + # NOTE: SeattleLabTable USD has its origin on the top surface, so the deformable object + # sits directly on it when placed at z = 0.05. + table: AssetBaseCfg = AssetBaseCfg( + prim_path="{ENV_REGEX_NS}/Table", + init_state=AssetBaseCfg.InitialStateCfg(pos=[0.5, 0.0, 0.0], rot=[0.0, 0.0, 0.707, 0.707]), + spawn=UsdFileCfg(usd_path=f"{ISAAC_NUCLEUS_DIR}/Props/Mounts/SeattleLabTable/table_instanceable.usd"), + ) + + # ground plane + ground: AssetBaseCfg = AssetBaseCfg( + prim_path="/World/GroundPlane", + init_state=AssetBaseCfg.InitialStateCfg(pos=[0.0, 0.0, -1.05]), + spawn=GroundPlaneCfg(), + ) + + # lights + # dome_light: AssetBaseCfg = AssetBaseCfg( + # prim_path="/World/light", + # spawn=sim_utils.DomeLightCfg(color=(0.75, 0.75, 0.75), intensity=2000.0), + # ) + sky_light = AssetBaseCfg( + prim_path="/World/skyLight", + spawn=sim_utils.DomeLightCfg( + intensity=750.0, + texture_file=f"{ISAAC_NUCLEUS_DIR}/Materials/Textures/Skies/PolyHaven/kloofendal_43d_clear_puresky_4k.hdr", + ), + ) + + def __post_init__(self) -> None: + # disable gravity on the arm so the low-PD actuators do not need to fight gravity sag, + # which is the dominant source of steady-state IK tracking error. + self.robot.spawn.rigid_props.disable_gravity = True + + # increase franka gripper stiffness + self.robot.actuators["panda_hand"].effort_limit_sim = 500.0 + self.robot.actuators["panda_hand"].stiffness = 1000.0 + self.robot.actuators["panda_hand"].damping = 100.0 + + +## +# MDP settings +## + + +@configclass +class CommandsCfg: + """Commands for the deformable goal pose (xyz + identity quat in robot root frame).""" + + deformable_pose = mdp.UniformPoseCommandCfg( + asset_name="robot", + body_name="panda_hand", + resampling_time_range=(5.0, 5.0), + debug_vis=True, + ranges=mdp.UniformPoseCommandCfg.Ranges( + pos_x=(0.4, 0.6), + pos_y=(-0.25, 0.25), + pos_z=(0.25, 0.5), + roll=(0.0, 0.0), + pitch=(0.0, 0.0), + yaw=(0.0, 0.0), + ), + # Render the goal as a transparent colored sphere (a point) instead of a coordinate frame. + goal_pose_visualizer_cfg=VisualizationMarkersCfg( + prim_path="/Visuals/Command/goal_pose", + markers={ + "sphere": sim_utils.SphereCfg( + radius=0.03, + visual_material=sim_utils.PreviewSurfaceCfg( + diffuse_color=(0.1, 0.9, 0.2), + opacity=0.4, + ), + ), + }, + ), + ) + + +@configclass +class ActionsCfg: + """7-dim absolute end-effector pose (xyz + quaternion) via differential IK + 1-dim binary gripper.""" + + arm_action = DifferentialInverseKinematicsActionCfg( + asset_name="robot", + joint_names=["panda_joint.*"], + body_name="panda_hand", + controller=DifferentialIKControllerCfg( + command_type="pose", + use_relative_mode=False, + ik_method="dls", + ik_params={"lambda_val": 0.6}, + ), + body_offset=DifferentialInverseKinematicsActionCfg.OffsetCfg(pos=[0.0, 0.0, 0.107]), + ) + gripper_action = mdp.BinaryJointPositionActionCfg( + asset_name="robot", + joint_names=["panda_finger.*"], + open_command_expr={"panda_finger_.*": 0.05}, + close_command_expr={"panda_finger_.*": 0.0}, + ) + + +@configclass +class ObservationsCfg: + """Policy observations: joint state, deformable COM in robot frame, target, last action.""" + + @configclass + class PolicyCfg(ObsGroup): + joint_pos = ObsTerm(func=mdp.joint_pos_rel) + joint_vel = ObsTerm(func=mdp.joint_vel_rel) + deformable_sampled_points = ObsTerm( + func=mdp.DeformableSampledPointsInRobotRootFrame, + params={"asset_cfg": SceneEntityCfg("deformable"), "num_points": 20}, + ) + target_position = ObsTerm(func=mdp.generated_commands, params={"command_name": "deformable_pose"}) + actions = ObsTerm(func=mdp.last_action) + + def __post_init__(self) -> None: + self.enable_corruption = True + self.concatenate_terms = True + + policy: PolicyCfg = PolicyCfg() + + +@configclass +class EventCfg: + """Reset events: robot to default joint config, deformable with small position randomization.""" + + reset_robot_joints = EventTerm( + func=mdp.reset_joints_by_scale, + mode="reset", + params={"position_range": (0.9, 1.1), "velocity_range": (0.0, 0.0)}, + ) + + reset_deformable = EventTerm( + func=mdp.reset_nodal_state_uniform, + mode="reset", + params={ + "position_range": {"x": (-0.05, 0.05), "y": (-0.05, 0.05), "z": (0.0, 0.0)}, + "velocity_range": {}, + "asset_cfg": SceneEntityCfg("deformable"), + }, + ) + + +@configclass +class RewardsCfg: + """Lift-to-target reward for a deformable object.""" + + reaching_deformable = RewTerm( + func=mdp.deformable_ee_distance, + params={"std": 0.1, "asset_cfg": SceneEntityCfg("deformable")}, + weight=5.0, + ) + lifting_deformable = RewTerm( + func=mdp.deformable_lifted, + params={"minimal_height": 0.04, "asset_cfg": SceneEntityCfg("deformable")}, + weight=5.0, + ) + deformable_goal_tracking = RewTerm( + func=mdp.deformable_com_goal_distance, + params={ + "std": 0.3, + "minimal_height": 0.075, + "command_name": "deformable_pose", + "asset_cfg": SceneEntityCfg("deformable"), + }, + weight=16.0, + ) + deformable_goal_tracking_fine_grained = RewTerm( + func=mdp.deformable_com_goal_distance, + params={ + "std": 0.05, + "minimal_height": 0.075, + "command_name": "deformable_pose", + "asset_cfg": SceneEntityCfg("deformable"), + }, + weight=5.0, + ) + + action_rate = RewTerm(func=mdp.action_rate_l2, weight=-1e-2) + gripper_close = RewTerm( + func=mdp.gripper_close_action, + params={"action_name": "gripper_action"}, + weight=-1.0, + ) + joint_vel = RewTerm(func=mdp.joint_vel_l2, weight=-1e-2) + joint_torque = RewTerm(func=mdp.joint_torques_l2, weight=-1e-4) + joint_acc = RewTerm(func=mdp.joint_acc_l2, weight=-1e-4) + + +@configclass +class TerminationsCfg: + """Time out + table bounds/drop termination.""" + + time_out = DoneTerm(func=mdp.time_out, time_out=True) + + deformable_outside_table = DoneTerm( + func=mdp.deformable_outside_table_bounds, + params={ + "x_bounds": (0.0, 1.0), + "y_bounds": (-0.5, 0.5), + "asset_cfg": SceneEntityCfg("deformable"), + }, + ) + + deformable_dropped = DoneTerm( + func=mdp.deformable_com_below_minimum, + params={"minimum_height": -0.1, "asset_cfg": SceneEntityCfg("deformable")}, + ) + + ee_below_table = DoneTerm( + func=mdp.ee_below_minimum, + params={"minimum_height": 0.0, "ee_frame_cfg": SceneEntityCfg("ee_frame")}, + ) + + +## +# Environment configuration +## + + +@configclass +class FrankaSoftSceneCfg(PresetCfg): + newton_mjwarp_vbd: _FrankaSoftSceneCfg = _FrankaSoftSceneCfg(num_envs=128, env_spacing=2.5, replicate_physics=True) + + # PhysX does not support replicating physics for deformable objects + physx: _FrankaSoftSceneCfg = _FrankaSoftSceneCfg(num_envs=128, env_spacing=2.5, replicate_physics=False) + + default = newton_mjwarp_vbd + + +@configclass +class FrankaSoftEnvCfg(ManagerBasedRLEnvCfg): + """Manager-based RL environment: Franka Panda lifting a volume deformable.""" + + # Scene settings + scene: FrankaSoftSceneCfg = FrankaSoftSceneCfg() + # Basic settings + observations: ObservationsCfg = ObservationsCfg() + actions: ActionsCfg = ActionsCfg() + commands: CommandsCfg = CommandsCfg() + # MDP settings + rewards: RewardsCfg = RewardsCfg() + terminations: TerminationsCfg = TerminationsCfg() + events: EventCfg = EventCfg() + + def __post_init__(self) -> None: + # general settings + self.decimation = 1 + self.episode_length_s = 5.0 + + # simulation settings + self.sim.dt = 1 / 60.0 + self.sim.render_interval = self.decimation + self.sim.gravity = (0.0, 0.0, 0.0) + self.sim.physics = PhysicsCfg() + + # viewer settings + self.viewer.origin_type = "asset_root" + self.viewer.asset_name = "robot" + self.viewer.env_index = 0 + self.viewer.eye = (1.25, -1.5, 0.75) + self.viewer.resolution = (1920, 1080) diff --git a/source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/lift_franka_soft/mdp/__init__.py b/source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/lift_franka_soft/mdp/__init__.py new file mode 100644 index 00000000000..8596e8efb4e --- /dev/null +++ b/source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/lift_franka_soft/mdp/__init__.py @@ -0,0 +1,10 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""This sub-module contains the functions that are specific to the environment.""" + +from isaaclab.utils.module import lazy_export + +lazy_export() diff --git a/source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/lift_franka_soft/mdp/__init__.pyi b/source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/lift_franka_soft/mdp/__init__.pyi new file mode 100644 index 00000000000..bb8b3af1985 --- /dev/null +++ b/source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/lift_franka_soft/mdp/__init__.pyi @@ -0,0 +1,28 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +__all__ = [ + "deformable_com_below_minimum", + "deformable_ee_distance", + "deformable_com_goal_distance", + "deformable_com_in_robot_root_frame", + "DeformableSampledPointsInRobotRootFrame", + "deformable_lifted", + "deformable_outside_table_bounds", + "ee_below_minimum", + "gripper_close_action", +] + +from .observations import DeformableSampledPointsInRobotRootFrame, deformable_com_in_robot_root_frame +from .rewards import ( + deformable_com_below_minimum, + deformable_ee_distance, + deformable_com_goal_distance, + deformable_lifted, + deformable_outside_table_bounds, + ee_below_minimum, + gripper_close_action, +) +from isaaclab.envs.mdp import * diff --git a/source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/lift_franka_soft/mdp/observations.py b/source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/lift_franka_soft/mdp/observations.py new file mode 100644 index 00000000000..a1aa2fa0aa6 --- /dev/null +++ b/source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/lift_franka_soft/mdp/observations.py @@ -0,0 +1,115 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Observation functions for the Franka deformable lifting environment.""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import TYPE_CHECKING + +import torch +import warp as wp + +from isaaclab.managers import ManagerTermBase, SceneEntityCfg +from isaaclab.utils.math import subtract_frame_transforms + +if TYPE_CHECKING: + from isaaclab.assets import Articulation, DeformableObject + from isaaclab.envs import ManagerBasedRLEnv + + +def deformable_com_in_robot_root_frame( + env: ManagerBasedRLEnv, + asset_cfg: SceneEntityCfg = SceneEntityCfg("deformable"), + robot_cfg: SceneEntityCfg = SceneEntityCfg("robot"), +) -> torch.Tensor: + """Position of the deformable object's COM in the robot's root frame [m]. + + The COM is the mean of the deformable's nodal positions (see + :attr:`~isaaclab.assets.DeformableObject.data.root_pos_w`). + + Returns: + Tensor of shape ``(num_envs, 3)``. + """ + asset: DeformableObject = env.scene[asset_cfg.name] + robot: Articulation = env.scene[robot_cfg.name] + com_w = wp.to_torch(asset.data.root_pos_w) + com_b, _ = subtract_frame_transforms(wp.to_torch(robot.data.root_pos_w), wp.to_torch(robot.data.root_quat_w), com_w) + return com_b + + +class DeformableSampledPointsInRobotRootFrame(ManagerTermBase): + """Sampled deformable nodal points expressed in the robot's root frame. + + The point indices are sampled on reset, then reused within the episode so + each observed point follows the same material node over time. + """ + + def __init__(self, cfg, env: ManagerBasedRLEnv): + super().__init__(cfg, env) + + self.asset_cfg: SceneEntityCfg = cfg.params.get("asset_cfg", SceneEntityCfg("deformable")) + self.robot_cfg: SceneEntityCfg = cfg.params.get("robot_cfg", SceneEntityCfg("robot")) + self.num_points: int = cfg.params.get("num_points", 20) + + asset: DeformableObject = env.scene[self.asset_cfg.name] + self.num_nodes = asset.data.nodal_pos_w.shape[1] + self.node_ids = torch.empty(env.num_envs, self.num_points, dtype=torch.long, device=env.device) + self.reset() + + def reset(self, env_ids: Sequence[int] | None = None) -> None: + """Resample observed deformable nodes for the selected environments.""" + if env_ids is None: + env_ids = slice(None) + num_envs = self.num_envs + else: + num_envs = len(env_ids) + + if self.num_points <= self.num_nodes: + self.node_ids[env_ids] = ( + torch.rand((num_envs, self.num_nodes), device=self.device).topk(self.num_points, dim=1).indices + ) + else: + self.node_ids[env_ids] = torch.randint(self.num_nodes, (num_envs, self.num_points), device=self.device) + + def __call__( + self, + env: ManagerBasedRLEnv, + asset_cfg: SceneEntityCfg = SceneEntityCfg("deformable"), + robot_cfg: SceneEntityCfg = SceneEntityCfg("robot"), + num_points: int = 20, + ) -> torch.Tensor: + """Sample deformable nodal positions in the robot's root frame. + + Args: + env: The environment instance. + asset_cfg: The deformable object entity. + robot_cfg: The robot entity providing the reference frame. + num_points: Number of sampled points. + + Returns: + Flattened tensor of shape ``(num_envs, 3 * num_points)`` with sampled + point positions [m] in the robot root frame. + """ + asset: DeformableObject = env.scene[asset_cfg.name] + robot: Articulation = env.scene[robot_cfg.name] + if num_points != self.num_points: + raise ValueError( + f"Requested {num_points} deformable points, but this term was initialized with {self.num_points}." + ) + + nodal_pos_w = asset.data.nodal_pos_w.torch + sampled_points_w = nodal_pos_w.gather(1, self.node_ids.unsqueeze(-1).expand(-1, -1, 3)) + + flat_sampled_points_w = sampled_points_w.reshape(-1, 3) + root_pos_w = robot.data.root_pos_w.torch.unsqueeze(1).expand(-1, num_points, -1) + root_quat_w = robot.data.root_quat_w.torch.unsqueeze(1).expand(-1, num_points, -1) + sampled_points_b, _ = subtract_frame_transforms( + root_pos_w.reshape(-1, 3), + root_quat_w.reshape(-1, 4), + flat_sampled_points_w, + ) + return sampled_points_b.view(env.num_envs, -1) diff --git a/source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/lift_franka_soft/mdp/rewards.py b/source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/lift_franka_soft/mdp/rewards.py new file mode 100644 index 00000000000..9811051cd37 --- /dev/null +++ b/source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/lift_franka_soft/mdp/rewards.py @@ -0,0 +1,167 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Reward and termination functions for the Franka deformable lifting environment.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import torch +import warp as wp + +from isaaclab.managers import SceneEntityCfg +from isaaclab.utils.math import combine_frame_transforms + +if TYPE_CHECKING: + from isaaclab.assets import Articulation, DeformableObject + from isaaclab.envs import ManagerBasedRLEnv + from isaaclab.sensors import FrameTransformer + + +def deformable_lifted( + env: ManagerBasedRLEnv, + minimal_height: float, + asset_cfg: SceneEntityCfg = SceneEntityCfg("deformable"), +) -> torch.Tensor: + """Reward if the deformable COM is above a minimum height. + + Args: + env: The environment instance. + minimal_height: Minimum COM height [m]. + asset_cfg: The deformable object entity. + + Returns: + Reward tensor with shape ``(num_envs,)``. + """ + asset: DeformableObject = env.scene[asset_cfg.name] + com_z = wp.to_torch(asset.data.root_pos_w)[:, 2] + return torch.where(com_z > minimal_height, 1.0, 0.0) + + +def deformable_ee_distance( + env: ManagerBasedRLEnv, + std: float, + asset_cfg: SceneEntityCfg = SceneEntityCfg("deformable"), + ee_frame_cfg: SceneEntityCfg = SceneEntityCfg("ee_frame"), +) -> torch.Tensor: + """Reward reaching the deformable's nearest nodal point with the end-effector. + + Args: + env: The environment instance. + std: The tanh kernel standard deviation [m]. + asset_cfg: The deformable object entity. + ee_frame_cfg: The end-effector frame entity. + + Returns: + Reward tensor with shape ``(num_envs,)``. + """ + asset: DeformableObject = env.scene[asset_cfg.name] + ee_frame: FrameTransformer = env.scene[ee_frame_cfg.name] + nodal_pos_w = wp.to_torch(asset.data.nodal_pos_w) + ee_w = wp.to_torch(ee_frame.data.target_pos_w)[..., 0, :] + distance = torch.linalg.norm(nodal_pos_w - ee_w.unsqueeze(1), dim=2).min(dim=1).values + return 1.0 - torch.tanh(distance / std) + + +def deformable_com_goal_distance( + env: ManagerBasedRLEnv, + std: float, + minimal_height: float, + command_name: str, + robot_cfg: SceneEntityCfg = SceneEntityCfg("robot"), + asset_cfg: SceneEntityCfg = SceneEntityCfg("deformable"), +) -> torch.Tensor: + """Reward tracking of the goal position by the deformable's COM (tanh kernel). + + Only credits when the COM is above ``minimal_height`` (i.e. the object is lifted). + The command is interpreted as ``[x, y, z, qw, qx, qy, qz]`` in the robot's root frame. + """ + robot: Articulation = env.scene[robot_cfg.name] + asset: DeformableObject = env.scene[asset_cfg.name] + command = env.command_manager.get_command(command_name) + des_pos_b = command[:, :3] + des_pos_w, _ = combine_frame_transforms( + wp.to_torch(robot.data.root_pos_w), wp.to_torch(robot.data.root_quat_w), des_pos_b + ) + com_w = wp.to_torch(asset.data.root_pos_w) + distance = torch.linalg.norm(des_pos_w - com_w, dim=1) + return (com_w[:, 2] > minimal_height) * (1.0 - torch.tanh(distance / std)) + + +def gripper_close_action(env: ManagerBasedRLEnv, action_name: str = "gripper_action") -> torch.Tensor: + """Penalty signal for commanding the gripper to close. + + The binary gripper action uses negative float actions for close commands and + non-negative actions for open commands. + + Args: + env: The environment instance. + action_name: Name of the gripper action term. + + Returns: + Tensor with shape ``(num_envs,)`` containing ``1`` when the gripper is + commanded closed and ``0`` otherwise. + """ + gripper_action = env.action_manager.get_term(action_name).raw_actions + return torch.any(gripper_action < 0.0, dim=1).float() + + +def deformable_com_below_minimum( + env: ManagerBasedRLEnv, + minimum_height: float, + asset_cfg: SceneEntityCfg = SceneEntityCfg("deformable"), +) -> torch.Tensor: + """Termination signal when the deformable's COM falls below ``minimum_height`` [m].""" + asset: DeformableObject = env.scene[asset_cfg.name] + com_z = wp.to_torch(asset.data.root_pos_w)[:, 2] + return com_z < minimum_height + + +def deformable_outside_table_bounds( + env: ManagerBasedRLEnv, + x_bounds: tuple[float, float], + y_bounds: tuple[float, float], + asset_cfg: SceneEntityCfg = SceneEntityCfg("deformable"), +) -> torch.Tensor: + """Terminate if any deformable nodal point leaves the table footprint. + + Args: + env: The environment instance. + x_bounds: Allowed x-position range in the environment frame [m]. + y_bounds: Allowed y-position range in the environment frame [m]. + asset_cfg: The deformable object entity. + + Returns: + Boolean tensor with shape ``(num_envs,)``. + """ + asset: DeformableObject = env.scene[asset_cfg.name] + nodal_pos = wp.to_torch(asset.data.nodal_pos_w) - env.scene.env_origins.unsqueeze(1) + outside_x = (nodal_pos[..., 0] < x_bounds[0]) | (nodal_pos[..., 0] > x_bounds[1]) + outside_y = (nodal_pos[..., 1] < y_bounds[0]) | (nodal_pos[..., 1] > y_bounds[1]) + return torch.any(outside_x | outside_y, dim=1) + + +def ee_below_minimum( + env: ManagerBasedRLEnv, + minimum_height: float, + ee_frame_cfg: SceneEntityCfg = SceneEntityCfg("ee_frame"), +) -> torch.Tensor: + """Termination signal when the end-effector falls below ``minimum_height`` [m]. + + Height is measured in the environment frame (``z`` of the EE position with the env + origin subtracted), so the threshold is independent of the environment's xy offset. + + Args: + env: The environment instance. + minimum_height: Minimum allowed EE height in the environment frame [m]. + ee_frame_cfg: The end-effector frame entity. + + Returns: + Boolean tensor with shape ``(num_envs,)``. + """ + ee_frame: FrameTransformer = env.scene[ee_frame_cfg.name] + ee_z = wp.to_torch(ee_frame.data.target_pos_w)[..., 0, 2] - env.scene.env_origins[:, 2] + return ee_z < minimum_height diff --git a/source/isaaclab_visualizers/changelog.d/daniela-move-scene-data.rst b/source/isaaclab_visualizers/changelog.d/daniela-move-scene-data.rst new file mode 100644 index 00000000000..ea709a9f96b --- /dev/null +++ b/source/isaaclab_visualizers/changelog.d/daniela-move-scene-data.rst @@ -0,0 +1,11 @@ +Fixed +^^^^^ + +* Updated ``configclass`` imports in :mod:`isaaclab_visualizers.kit`, + :mod:`isaaclab_visualizers.newton`, :mod:`isaaclab_visualizers.rerun`, and + :mod:`isaaclab_visualizers.viser` visualizer configs to import from + :mod:`isaaclab.utils.configclass` directly, matching the lazy-import layout + introduced in :mod:`isaaclab.utils`. +* Updated ``test_visualizer_cartpole_integration`` to read the tiled camera + RGB output via the ``.torch`` accessor, matching the Warp-backed camera + data API. diff --git a/source/isaaclab_visualizers/changelog.d/mtrepte-update-debug-viz.rst b/source/isaaclab_visualizers/changelog.d/mtrepte-update-debug-viz.rst new file mode 100644 index 00000000000..f2e45fe04ed --- /dev/null +++ b/source/isaaclab_visualizers/changelog.d/mtrepte-update-debug-viz.rst @@ -0,0 +1,7 @@ +Changed +^^^^^^^ + +* Changed Rerun and Viser visualizers to avoid opening browser tabs by default and to show browser URLs in the startup logs instead. +* Changed visualizer initialization tables to debug-level logging to reduce default startup log noise. +* Added non-interactive tiled camera image views for Kit and Newton visualizers, with generated per-env cameras or existing Camera sensor support. +* Added clearer Kit visualizer errors when tiled camera views are enabled without camera rendering support. diff --git a/source/isaaclab_visualizers/isaaclab_visualizers/kit/kit_visualizer.py b/source/isaaclab_visualizers/isaaclab_visualizers/kit/kit_visualizer.py index 4766fbe7f68..6ea630f7aed 100644 --- a/source/isaaclab_visualizers/isaaclab_visualizers/kit/kit_visualizer.py +++ b/source/isaaclab_visualizers/isaaclab_visualizers/kit/kit_visualizer.py @@ -11,9 +11,25 @@ import logging from typing import TYPE_CHECKING +import numpy as np +import torch + from pxr import Gf, Usd, UsdGeom, Vt from isaaclab.app.settings_manager import get_settings_manager +from isaaclab.envs.utils.camera_view import ( + VISUALIZER_TILED_CAMERA_MAX_TILES, + apply_camera_target_positions, + camera_rgb_batch, + compose_rgb_grid_tensor, + compute_tile_resolution, + create_visualizer_camera, + find_camera_by_prim_path, + prim_world_positions, + remove_generated_prims, + resolve_tiled_env_indices, +) +from isaaclab.utils.math import create_rotation_matrix_from_view, quat_from_matrix from isaaclab.visualizers.base_visualizer import BaseVisualizer from isaaclab_visualizers.newton_adapter import resolve_visible_env_indices @@ -23,7 +39,7 @@ logger = logging.getLogger(__name__) if TYPE_CHECKING: - from isaaclab.scene.scene_data_provider import SceneDataProvider + from isaaclab.scene_data import SceneDataProvider _DEFAULT_VIEWPORT_NAME = "Visualizer Viewport" @@ -46,12 +62,26 @@ def __init__(self, cfg: KitVisualizerCfg): self._is_initialized = False self._sim_time = 0.0 self._step_counter = 0 + self._env_ids = None + self._resolved_visible_env_ids: list[int] | None = None self._hidden_env_visibilities: dict[str, str] = {} # PointInstancer prim path -> (had authored invisibleIds, previous value) for partial viz restore. self._point_instancer_invisible_ids_backup: dict[str, tuple[bool, object]] = {} self._runtime_headless = bool(cfg.headless) # USD path for the viewport's active camera, refreshed after setup (used by CI/tests). self._controlled_camera_path: str | None = None + self._camera_sensor = None + self._camera_sensor_indices: list[int] = [] + self._camera_env_indices: list[int] = [] + self._camera_is_owned = False + self._generated_camera_prim_paths: list[str] = [] + self._generated_camera_xform_ops: dict[str, tuple[UsdGeom.XformOp, UsdGeom.XformOp]] = {} + self._generated_camera_pose_cache: dict[str, tuple[float, ...]] = {} + self._generated_camera_poses_dirty = False + self._camera_image_provider = None + self._camera_image_window = None + self._camera_gpu_upload_tensor = None + self._warned_gpu_upload_failure = False # ---- Lifecycle ------------------------------------------------------------------------ @@ -65,9 +95,7 @@ def initialize(self, scene_data_provider: SceneDataProvider) -> None: logger.debug("[KitVisualizer] initialize() called while already initialized.") return - if scene_data_provider is None: - raise RuntimeError("[KitVisualizer] Requires a scene_data_provider.") - self._scene_data_provider = scene_data_provider + scene_data_provider = self._set_scene_data_provider(scene_data_provider) usd_stage = scene_data_provider.usd_stage if usd_stage is None: raise RuntimeError("[KitVisualizer] USD stage not available from scene_data_provider.") @@ -92,13 +120,15 @@ def initialize(self, scene_data_provider: SceneDataProvider) -> None: rows=[ ("eye", self.cfg.eye), ("lookat", self.cfg.lookat), - ("cam_source", self.cfg.cam_source), + ("tiled_cam_view", self.cfg.tiled_cam_view), + ("tiled_cam_num", self.cfg.tiled_cam_num), ("max_visible_envs", self.cfg.max_visible_envs), ("num_visualized_envs", num_visualized_envs), ("create_viewport", self.cfg.create_viewport), ("headless", self._runtime_headless), ], ) + self._setup_camera_sensor_view(num_envs) self._is_initialized = True @@ -125,6 +155,7 @@ def step(self, dt: float) -> None: settings.set_bool("/app/player/playSimulations", True) except (ImportError, AttributeError) as exc: logger.debug("[KitVisualizer] App update skipped: %s", exc) + self._update_camera_image_panel(dt) # Markers (VisualizationMarkers) are often created or resized to num_envs only after the first # simulation / debug-vis step; re-apply PointInstancer invisibleIds each step when partial viz is on. self._refresh_partial_viz_point_instancers_if_needed() @@ -134,6 +165,14 @@ def close(self) -> None: if not self._is_initialized: return self._restore_env_visibility() + if self._camera_sensor is not None and self._camera_is_owned: + remove_generated_prims(self._generated_camera_prim_paths) + self._camera_sensor = None + self._generated_camera_xform_ops.clear() + self._generated_camera_pose_cache.clear() + self._generated_camera_poses_dirty = False + self._camera_image_provider = None + self._camera_image_window = None self._simulation_app = None self._viewport_window = None self._viewport_api = None @@ -188,17 +227,10 @@ def set_camera_view( ) -> None: """Set active viewport camera eye/target. - When :attr:`self.cfg.cam_source` is ``"cfg"``, this is a no-op: the pose comes only from - :attr:`self.cfg.eye` / :attr:`self.cfg.lookat` (applied in :meth:`_setup_viewport`). Otherwise - :class:`~isaaclab.sim.simulation_context.SimulationContext` and :class:`ViewportCameraController` - would overwrite that pose with :class:`~isaaclab.envs.common.ViewerCfg`-driven views. - Args: eye: Camera eye position. target: Camera look-at target. """ - if self.cfg.cam_source == "cfg": - return if not self._is_initialized: logger.debug("[KitVisualizer] set_camera_view() ignored because visualizer is not initialized.") return @@ -240,12 +272,10 @@ def _setup_viewport(self) -> None: # Headless: no viewport window; apply cfg pose to the default perspective camera path. self._viewport_window = None self._viewport_api = None - if self.cfg.cam_source == "prim_path": - logger.warning( - "[KitVisualizer] cam_source='prim_path' has limited support in headless mode; " - "using eye/lookat from cfg instead." - ) - self._apply_cfg_camera_pose_if_configured() + if self._uses_camera_sensor_view(): + logger.debug("[KitVisualizer] Camera image view requested in headless mode; no UI panel is created.") + else: + self._apply_cfg_camera_pose_if_configured() self._refresh_controlled_camera_path() return @@ -286,16 +316,187 @@ def _setup_viewport(self) -> None: self._refresh_controlled_camera_path() return self._viewport_api = self._viewport_window.viewport_api - if self.cfg.cam_source == "prim_path": - if not self._set_active_camera_path(self.cfg.cam_prim_path): - raise RuntimeError( - "[KitVisualizer] cam_source='prim_path' requires a valid cam_prim_path. " - f"Camera prim not found: '{self.cfg.cam_prim_path}'." - ) + if self._uses_camera_sensor_view(): + # Camera sensor image views are shown in a non-interactive image panel. + pass else: self._apply_cfg_camera_pose_if_configured() self._refresh_controlled_camera_path() + def _uses_camera_sensor_view(self) -> bool: + """Return whether Kit should display a camera sensor image instead of an interactive viewport camera.""" + return bool(self.cfg.tiled_cam_view) + + def _setup_camera_sensor_view(self, num_envs: int) -> None: + """Resolve or create the Camera sensor backing non-interactive image views.""" + if not self._uses_camera_sensor_view(): + return + if self._runtime_headless: + return + if not get_settings_manager().get("/isaaclab/cameras_enabled", False): + raise RuntimeError( + "[KitVisualizer] tiled_cam_view=True requires camera rendering support. " + "Rerun with --enable_cameras, or disable tiled_cam_view for this visualizer config." + ) + logger.debug( + "[KitVisualizer] Setting up camera image view: tiled=%s source=%s num_envs=%s", + self.cfg.tiled_cam_view, + "prim_path" if self.cfg.tiled_cam_prim_path is not None else "generated", + num_envs, + ) + env_ids = resolve_tiled_env_indices( + num_envs, + self.cfg.tiled_cam_num, + self.cfg.tiled_cam_env_indices, + max_tiles=VISUALIZER_TILED_CAMERA_MAX_TILES, + sample_from=self._resolved_visible_env_ids, + ) + self._camera_env_indices = env_ids + if self.cfg.tiled_cam_prim_path is not None: + logger.debug( + "[KitVisualizer] tiled_cam_prim_path uses existing camera sensor output; " + "generated tiled camera pose fields are ignored." + ) + cameras = self._scene_data_provider.get_camera_sensors() + self._camera_sensor = find_camera_by_prim_path(cameras, self.cfg.tiled_cam_prim_path, env_ids) + self._camera_sensor_indices = env_ids + else: + from isaaclab_physx.renderers import IsaacRtxRendererCfg + + count = max(1, len(env_ids)) + tile_w, tile_h = compute_tile_resolution(self.cfg.window_width, self.cfg.window_height, count) + logger.debug( + "[KitVisualizer] Creating generated camera sensor: env_ids=%s tile=%sx%s", + env_ids, + tile_w, + tile_h, + ) + self._camera_sensor, self._generated_camera_prim_paths = create_visualizer_camera( + num_envs=num_envs, + width=tile_w, + height=tile_h, + renderer_cfg=IsaacRtxRendererCfg(), + ) + logger.debug("[KitVisualizer] Generated camera sensor initialized.") + self._camera_sensor_indices = env_ids + self._camera_is_owned = True + self._update_owned_camera_poses() + logger.debug("[KitVisualizer] Generated camera poses initialized.") + self._setup_camera_image_window() + logger.debug("[KitVisualizer] Camera image window initialized.") + + def _setup_camera_image_window(self) -> None: + """Create a dockable Kit UI image panel for camera sensor RGB output.""" + import omni.ui + + title = self.cfg.viewport_name or "Visualizer Tiled Camera" + self._camera_image_provider = omni.ui.ByteImageProvider() + self._camera_image_window = omni.ui.Window(title, width=self.cfg.window_width, height=self.cfg.window_height) + with self._camera_image_window.frame: + omni.ui.ImageWithProvider(self._camera_image_provider) + + dock_position_name = self.cfg.dock_position.upper() + dock_position_map = { + "LEFT": omni.ui.DockPosition.LEFT, + "RIGHT": omni.ui.DockPosition.RIGHT, + "BOTTOM": omni.ui.DockPosition.BOTTOM, + "SAME": omni.ui.DockPosition.SAME, + } + asyncio.ensure_future( + self._dock_image_window_async(title, dock_position_map.get(dock_position_name, omni.ui.DockPosition.SAME)) + ) + + async def _dock_image_window_async(self, window_name: str, dock_position) -> None: + """Dock the camera image panel next to the main viewport.""" + import omni.kit.app + import omni.ui + + image_window = None + for _ in range(10): + image_window = omni.ui.Workspace.get_window(window_name) + if image_window: + break + await omni.kit.app.get_app().next_update_async() + main_viewport = omni.ui.Workspace.get_window("Viewport") + if image_window is not None and main_viewport is not None and image_window != main_viewport: + image_window.dock_in(main_viewport, dock_position, 0.5) + + def _update_owned_camera_poses(self) -> None: + """Update generated camera poses from env origins or follow prims.""" + if self._camera_sensor is None or not self._camera_is_owned: + return + target_positions = prim_world_positions( + self._scene_data_provider.get_usd_stage(), + self.cfg.tiled_cam_target_prim_path, + self._camera_env_indices, + scene=self._scene_data_provider.get_interactive_scene(), + ) + eyes, targets = apply_camera_target_positions( + self._camera_sensor, target_positions, self.cfg.tiled_cam_eye, self._camera_env_indices + ) + self._set_generated_usd_camera_poses(eyes, targets) + + def _update_camera_image_panel(self, dt: float) -> None: + """Refresh the non-interactive Kit image panel from camera RGB output.""" + if self._camera_sensor is None or self._camera_image_provider is None: + return + if self._camera_is_owned: + self._update_owned_camera_poses() + if self._generated_camera_poses_dirty: + self._sync_camera_pose_updates_to_kit() + self._generated_camera_poses_dirty = False + if self._camera_is_owned: + self._camera_sensor.update(dt=dt, force_recompute=True) + rgb = camera_rgb_batch(self._camera_sensor, self._camera_sensor_indices) + image = compose_rgb_grid_tensor(rgb) if self.cfg.tiled_cam_view else rgb[0].contiguous() + self._upload_camera_image_to_panel(image) + + def _upload_camera_image_to_panel(self, image: np.ndarray | torch.Tensor) -> None: + """Upload an RGB/RGBA image to the Kit image provider.""" + if isinstance(image, torch.Tensor): + if image.is_cuda: + try: + import omni.gpu_foundation_factory as gf + + if image.ndim == 3 and image.shape[2] == 3: + alpha = torch.full((*image.shape[:2], 1), 255, dtype=torch.uint8, device=image.device) + image = torch.cat((image, alpha), dim=2) + image = image.to(dtype=torch.uint8).contiguous() + self._camera_gpu_upload_tensor = image + self._camera_image_provider.set_bytes_data_from_gpu( + int(image.data_ptr()), [int(image.shape[1]), int(image.shape[0])], gf.TextureFormat.RGBA8_UNORM + ) + return + except Exception as exc: + if not self._warned_gpu_upload_failure: + logger.warning("[KitVisualizer] GPU image upload failed; falling back to CPU upload: %s", exc) + self._warned_gpu_upload_failure = True + image = image.detach().contiguous().cpu().numpy() + + image = image.astype("uint8", copy=False) + if image.ndim == 3 and image.shape[2] == 3: + alpha = np.full((*image.shape[:2], 1), 255, dtype=np.uint8) + image = np.concatenate((image, alpha), axis=2) + image = np.ascontiguousarray(image) + self._camera_image_provider.set_bytes_data(image.flatten().data, [image.shape[1], image.shape[0]]) + + def _sync_camera_pose_updates_to_kit(self) -> None: + """Flush generated camera pose writes before camera RGB is sampled.""" + try: + import omni.kit.app + + app = omni.kit.app.get_app() + if app is None or not app.is_running(): + return + settings = get_settings_manager() + play_flag = settings.get("/app/player/playSimulations") + settings.set_bool("/app/player/playSimulations", False) + app.update() + if play_flag is not None: + settings.set_bool("/app/player/playSimulations", bool(play_flag)) + except Exception as exc: + logger.debug("[KitVisualizer] Camera pose Kit sync skipped: %s", exc) + def _refresh_controlled_camera_path(self) -> None: """Cache :attr:`_controlled_camera_path` from the active viewport (or default persp).""" if self._viewport_api is not None: @@ -354,6 +555,74 @@ def _set_viewport_camera(self, position: tuple[float, float, float], target: tup camera_state.set_position_world(Gf.Vec3d(float(position[0]), float(position[1]), float(position[2])), True) camera_state.set_target_world(Gf.Vec3d(float(target[0]), float(target[1]), float(target[2])), True) + def _set_generated_usd_camera_poses(self, eyes: torch.Tensor, targets: torch.Tensor) -> None: + """Author generated camera poses directly on USD camera prims for Kit/Fabric visibility.""" + # TODO: Remove this USD-side pose path once Fabric-backed camera transforms propagate reliably to Kit. + for local_idx, env_id in enumerate(self._camera_env_indices): + if local_idx >= eyes.shape[0]: + break + camera_path = ( + self._generated_camera_prim_paths[env_id] + if 0 <= env_id < len(self._generated_camera_prim_paths) + else f"/World/envs/env_{env_id}/VisualizerCamera" + ) + self._generated_camera_poses_dirty |= self._set_usd_camera_pose( + camera_path, eyes[local_idx], targets[local_idx] + ) + + def _set_usd_camera_pose(self, camera_path: str, position, target) -> bool: + """Apply eye/target camera pose directly to a USD camera prim. + + Returns: + ``True`` when authored values changed, otherwise ``False``. + """ + # TODO: Remove this USD-side pose path once Fabric-backed camera transforms propagate reliably to Kit. + usd_stage = self._scene_data_provider.usd_stage if self._scene_data_provider else None + if usd_stage is None: + return False + + eye = torch.as_tensor(position, dtype=torch.float32, device="cpu").reshape(1, 3) + lookat = torch.as_tensor(target, dtype=torch.float32, device="cpu").reshape(1, 3) + up_axis = UsdGeom.GetStageUpAxis(usd_stage) + rotation_matrix = create_rotation_matrix_from_view(eye, lookat, up_axis=up_axis, device="cpu") + if torch.isnan(rotation_matrix).any(): + raise ValueError("[KitVisualizer] Cannot set camera pose because eye and lookat are degenerate.") + quat_xyzw = quat_from_matrix(rotation_matrix)[0] + pose_key = ( + float(eye[0, 0]), + float(eye[0, 1]), + float(eye[0, 2]), + float(quat_xyzw[0]), + float(quat_xyzw[1]), + float(quat_xyzw[2]), + float(quat_xyzw[3]), + ) + if self._generated_camera_pose_cache.get(camera_path) == pose_key: + return False + + if camera_path not in self._generated_camera_xform_ops: + camera = UsdGeom.Camera.Define(usd_stage, camera_path) + camera_xform = UsdGeom.Xformable(camera.GetPrim()) + camera_xform.ClearXformOpOrder() + # Generated visualizer cameras live under env prims, but eyes/targets are world-space. + # Reset the xform stack so Kit/Fabric sees the authored pose as a world pose. + camera_xform.SetResetXformStack(True) + translate_op = camera_xform.AddTranslateOp() + orient_op = camera_xform.AddOrientOp(UsdGeom.XformOp.PrecisionDouble) + self._generated_camera_xform_ops[camera_path] = (translate_op, orient_op) + else: + translate_op, orient_op = self._generated_camera_xform_ops[camera_path] + + quat_gf = Gf.Quatd( + float(quat_xyzw[3]), + Gf.Vec3d(float(quat_xyzw[0]), float(quat_xyzw[1]), float(quat_xyzw[2])), + ) + + translate_op.Set(Gf.Vec3d(float(eye[0, 0]), float(eye[0, 1]), float(eye[0, 2]))) + orient_op.Set(quat_gf) + self._generated_camera_pose_cache[camera_path] = pose_key + return True + def _apply_cfg_camera_pose_if_configured(self) -> None: """Apply configured camera pose from eye/lookat.""" self._set_viewport_camera(self.cfg.eye, self.cfg.lookat) diff --git a/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer.py b/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer.py index 31c17a7b16d..a65c6f78200 100644 --- a/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer.py +++ b/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer.py @@ -8,15 +8,28 @@ from __future__ import annotations import logging +import math +import os from typing import TYPE_CHECKING -import numpy as np import warp as wp from newton.viewer import ViewerGL from pyglet.math import Vec3 as PygletVec3 +from isaaclab.envs.utils.camera_view import ( + VISUALIZER_TILED_CAMERA_MAX_TILES, + apply_camera_target_positions, + camera_rgb_batch, + compute_tile_resolution, + create_visualizer_camera, + find_camera_by_prim_path, + prim_world_positions, + remove_generated_prims, + resolve_tiled_env_indices, +) from isaaclab.visualizers.base_visualizer import BaseVisualizer +from isaaclab_visualizers.newton.newton_visualization_markers import render_newton_visualization_markers from isaaclab_visualizers.newton_adapter import apply_viewer_visible_worlds, resolve_visible_env_indices from .newton_visualizer_cfg import NewtonVisualizerCfg @@ -24,7 +37,7 @@ logger = logging.getLogger(__name__) if TYPE_CHECKING: - from isaaclab.scene.scene_data_provider import SceneDataProvider + from isaaclab.scene_data import SceneDataProvider class NewtonViewerGL(ViewerGL): @@ -226,6 +239,11 @@ def _render_left_panel(self): except Exception as exc: logger.debug("[NewtonVisualizer] Rendering color controls failed: %s", exc) + # Newton's ImageLogger owns camera-output image windows. Since Isaac Lab overrides + # ViewerGL's left panel, explicitly keep the logged-image selector and draw path. + if self._image_logger is not None: + self._image_logger.draw_controls() + imgui.set_next_item_open(True, imgui.Cond_.appearing) if imgui.collapsing_header("Camera"): imgui.separator() @@ -246,8 +264,51 @@ def _render_left_panel(self): imgui.text("ESC - Exit") imgui.end() + if self._image_logger is not None: + self._prime_image_logger_window_layout() + self._image_logger.draw() return + def _prime_image_logger_window_layout(self) -> None: + """Make first-open image windows use the available viewer space. + + Newton's ImageLogger initializes windows from a fixed per-tile size, so + small tiled batches otherwise open as small windows. Prime the next + ImGui window size once, then let users move/resize it normally. + """ + image_logger = self._image_logger + if image_logger is None: + return + selected = image_logger._selected + if selected is None: + return + entry = image_logger._images.get(selected) + if entry is None or entry.window_initialized: + return + + imgui = self.ui.imgui + viewport = imgui.get_main_viewport() + sidebar_width = float(image_logger._sidebar_width_px) + margin = 20.0 + available_w = max(320.0, viewport.work_size.x - sidebar_width - 2.0 * margin) + available_h = max(240.0, viewport.work_size.y - 2.0 * margin) + + n_tiles = max(1, int(entry.n)) + tile_aspect = float(entry.tile_aspect) + cols = max(1, math.ceil(math.sqrt(n_tiles))) + rows = math.ceil(n_tiles / cols) + grid_aspect = (rows * tile_aspect) / cols + title_and_padding_h = 40.0 + + window_w = available_w + window_h = min(available_h, max(240.0, window_w * grid_aspect + title_and_padding_h)) + pos_x = sidebar_width + margin + pos_y = margin + + imgui.set_next_window_pos(imgui.ImVec2(float(pos_x), float(pos_y)), imgui.Cond_.once) + imgui.set_next_window_size(imgui.ImVec2(float(window_w), float(window_h)), imgui.Cond_.once) + entry.window_initialized = True + class NewtonVisualizer(BaseVisualizer): """Newton OpenGL visualizer for Isaac Lab.""" @@ -266,10 +327,14 @@ def __init__(self, cfg: NewtonVisualizerCfg): self._model = None self._state = None self._update_frequency = cfg.update_frequency - self._scene_data_provider = None self._last_camera_pose: tuple[tuple[float, float, float], tuple[float, float, float]] | None = None self._headless_no_viewer = False self._resolved_visible_env_ids: list[int] | None = None + self._camera_sensor = None + self._camera_sensor_indices: list[int] = [] + self._camera_env_indices: list[int] = [] + self._camera_is_owned = False + self._generated_camera_prim_paths: list[str] = [] def initialize(self, scene_data_provider: SceneDataProvider) -> None: """Initialize viewer resources and bind scene data provider. @@ -282,19 +347,20 @@ def initialize(self, scene_data_provider: SceneDataProvider) -> None: if self._is_initialized: logger.debug("[NewtonVisualizer] initialize() called while already initialized.") return - if scene_data_provider is None: - raise RuntimeError("Newton visualizer requires a scene_data_provider.") - self._scene_data_provider = scene_data_provider + scene_data_provider = self._set_scene_data_provider(scene_data_provider) num_envs = scene_data_provider.num_envs metadata = {"num_envs": num_envs} self._env_ids = self._compute_visualized_env_ids() self._model = NewtonManager.get_model() - self._state = NewtonManager.get_state() + self._state = NewtonManager.get_state(self._scene_data_provider) - # Use pyglet's EGL headless backend when requested. Must run before the first - # ``pyglet.window`` import so ``Window`` resolves to :class:`~pyglet.window.headless.HeadlessWindow`. - if self.cfg.headless: + runtime_headless = self.cfg.headless or not os.environ.get("DISPLAY") + + # Use pyglet's EGL headless backend when requested or when no X display is available. + # This must run before the first ``pyglet.window`` import so ``Window`` resolves to + # :class:`~pyglet.window.headless.HeadlessWindow`. + if runtime_headless: import pyglet pyglet.options["headless"] = True @@ -302,7 +368,7 @@ def initialize(self, scene_data_provider: SceneDataProvider) -> None: self._viewer = NewtonViewerGL( width=self.cfg.window_width, height=self.cfg.window_height, - headless=self.cfg.headless, + headless=runtime_headless, metadata=metadata, update_frequency=self.cfg.update_frequency, ) @@ -316,6 +382,7 @@ def initialize(self, scene_data_provider: SceneDataProvider) -> None: num_envs=num_envs, ) self._viewer.set_world_offsets((0.0, 0.0, 0.0)) + self._apply_camera_focal_length() initial_pose = self._resolve_initial_camera_pose() self._apply_camera_pose(initial_pose) self._viewer.up_axis = 2 # Z-up @@ -340,6 +407,7 @@ def initialize(self, scene_data_provider: SceneDataProvider) -> None: self._viewer.renderer._light_color = self._viewer._coerce_color3(self.cfg.light_color) self._resolved_visible_env_ids = resolve_visible_env_indices(self._env_ids, self.cfg.max_visible_envs, num_envs) + self._setup_camera_sensor_view(num_envs) num_visualized_envs = ( len(self._resolved_visible_env_ids) if self._resolved_visible_env_ids is not None else num_envs ) @@ -352,7 +420,9 @@ def initialize(self, scene_data_provider: SceneDataProvider) -> None: tuple(float(x) for x in self._viewer.camera.pos) if self._viewer is not None else self.cfg.eye, ), ("lookat", self._last_camera_pose[1] if self._last_camera_pose else self.cfg.lookat), - ("cam_source", self.cfg.cam_source), + ("focal_length", self.cfg.focal_length), + ("tiled_cam_view", self.cfg.tiled_cam_view), + ("tiled_cam_num", self.cfg.tiled_cam_num), ("num_visualized_envs", num_visualized_envs), ("headless", self.cfg.headless), ], @@ -374,32 +444,37 @@ def step(self, dt: float) -> None: from isaaclab_newton.physics import NewtonManager if self._viewer is None: - self._state = NewtonManager.get_state() + self._state = NewtonManager.get_state(self._scene_data_provider) return - if self.cfg.cam_source == "prim_path": - self._update_camera_from_usd_path() - - self._state = NewtonManager.get_state() + self._state = NewtonManager.get_state(self._scene_data_provider) update_frequency = self._viewer._update_frequency if self._viewer else self._update_frequency if self._step_counter % update_frequency != 0: return + num_envs = NewtonManager.get_num_envs() + try: if not self._viewer.is_paused(): self._viewer.begin_frame(self._sim_time) - if self._state is not None: - body_q = getattr(self._state, "body_q", None) - if hasattr(body_q, "shape") and body_q.shape[0] == 0: - self._viewer.end_frame() - return - self._viewer.log_state(self._state) - self._viewer.end_frame() + try: + if self._state is not None: + body_q = getattr(self._state, "body_q", None) + if hasattr(body_q, "shape") and body_q.shape[0] == 0: + return + self._viewer.log_state(self._state) + if self.cfg.enable_markers: + render_newton_visualization_markers( + self._viewer, self._resolved_visible_env_ids, num_envs=num_envs + ) + self._log_camera_sensor_image() + finally: + self._viewer.end_frame() else: self._viewer._update() - except Exception as exc: - logger.debug("[NewtonVisualizer] Viewer update failed: %s", exc) + except Exception: + logger.exception("[NewtonVisualizer] Viewer update failed.") def close(self) -> None: """Release viewer resources.""" @@ -407,6 +482,9 @@ def close(self) -> None: return if self._viewer is not None: self._viewer = None + if self._camera_sensor is not None and self._camera_is_owned: + remove_generated_prims(self._generated_camera_prim_paths) + self._camera_sensor = None self._is_closed = True def is_running(self) -> bool: @@ -429,16 +507,73 @@ def _resolve_initial_camera_pose(self) -> tuple[tuple[float, float, float], tupl Returns: Camera eye and target tuples. """ - if self.cfg.cam_source == "prim_path": - pose = self._resolve_camera_pose_from_usd_path(self.cfg.cam_prim_path) - if pose is not None: - return pose - raise RuntimeError( - "[NewtonVisualizer] cam_source='prim_path' requires a resolvable camera prim path, " - f"but no camera pose was found for '{self.cfg.cam_prim_path}'." - ) return self._resolve_cfg_camera_pose("NewtonVisualizer") + def _uses_camera_sensor_view(self) -> bool: + """Return whether the visualizer displays camera sensor images instead of interactive camera controls.""" + return bool(self.cfg.tiled_cam_view) + + def _setup_camera_sensor_view(self, num_envs: int) -> None: + """Resolve or create the camera sensor used by non-interactive image views.""" + if not self._uses_camera_sensor_view(): + return + env_ids = resolve_tiled_env_indices( + num_envs, + self.cfg.tiled_cam_num, + self.cfg.tiled_cam_env_indices, + max_tiles=VISUALIZER_TILED_CAMERA_MAX_TILES, + sample_from=self._resolved_visible_env_ids, + ) + self._camera_env_indices = env_ids + if self.cfg.tiled_cam_prim_path is not None: + logger.debug( + "[NewtonVisualizer] tiled_cam_prim_path uses existing camera sensor output; " + "generated tiled camera pose fields are ignored." + ) + cameras = self._scene_data_provider.get_camera_sensors() + self._camera_sensor = find_camera_by_prim_path(cameras, self.cfg.tiled_cam_prim_path, env_ids) + self._camera_sensor_indices = env_ids + return + + from isaaclab_newton.renderers import NewtonWarpRendererCfg + + count = max(1, len(env_ids)) + tile_w, tile_h = compute_tile_resolution(self.cfg.window_width, self.cfg.window_height, count) + self._camera_sensor, self._generated_camera_prim_paths = create_visualizer_camera( + num_envs=num_envs, + width=tile_w, + height=tile_h, + renderer_cfg=NewtonWarpRendererCfg(), + ) + self._camera_sensor_indices = env_ids + self._camera_is_owned = True + self._update_owned_camera_poses() + + def _update_owned_camera_poses(self) -> None: + """Update generated camera poses from env origins or follow prims.""" + if self._camera_sensor is None or not self._camera_is_owned: + return + target_positions = prim_world_positions( + self._scene_data_provider.get_usd_stage(), + self.cfg.tiled_cam_target_prim_path, + self._camera_env_indices, + scene=self._scene_data_provider.get_interactive_scene(), + ) + eyes, targets = apply_camera_target_positions( + self._camera_sensor, target_positions, self.cfg.tiled_cam_eye, self._camera_env_indices + ) + + def _log_camera_sensor_image(self) -> None: + """Log the selected camera sensor RGB output into Newton's image panel.""" + if self._viewer is None or self._camera_sensor is None: + return + if self._camera_is_owned: + self._update_owned_camera_poses() + if self._camera_is_owned: + self._camera_sensor.update(dt=0.0, force_recompute=True) + rgb = camera_rgb_batch(self._camera_sensor, self._camera_sensor_indices).contiguous() + self._viewer.log_image("Visualizer Tiled Camera", wp.from_torch(rgb)) + def _apply_camera_pose(self, pose: tuple[tuple[float, float, float], tuple[float, float, float]]) -> None: """Apply camera eye/target pose to the Newton viewer. @@ -450,24 +585,14 @@ def _apply_camera_pose(self, pose: tuple[tuple[float, float, float], tuple[float cam_pos, cam_target = pose # Match Newton's Camera native pos type: PyVec3, not wp.vec3. self._viewer.camera.pos = PygletVec3(*cam_pos) - cam_pos_np = np.array(cam_pos, dtype=np.float32) - cam_target_np = np.array(cam_target, dtype=np.float32) - direction = cam_target_np - cam_pos_np - yaw = np.degrees(np.arctan2(direction[1], direction[0])) - horizontal_dist = np.sqrt(direction[0] ** 2 + direction[1] ** 2) - pitch = np.degrees(np.arctan2(direction[2], horizontal_dist)) - self._viewer.camera.yaw = float(yaw) - self._viewer.camera.pitch = float(pitch) + self._viewer.camera.look_at(cam_target) self._last_camera_pose = (cam_pos, cam_target) - def _update_camera_from_usd_path(self) -> None: - """Refresh camera pose from configured USD camera path when it changes.""" - pose = self._resolve_camera_pose_from_usd_path(self.cfg.cam_prim_path) - if pose is None: - return - if self._last_camera_pose == pose: + def _apply_camera_focal_length(self) -> None: + """Apply cfg focal length to Newton's vertical-FOV camera.""" + if self._viewer is None: return - self._apply_camera_pose(pose) + self._viewer.camera.fov = self._focal_length_to_vertical_fov_degrees() def supports_markers(self) -> bool: """Newton OpenGL viewer supports Isaac Lab markers through viewer-side meshes and lines.""" diff --git a/source/isaaclab_visualizers/isaaclab_visualizers/rerun/rerun_visualizer.py b/source/isaaclab_visualizers/isaaclab_visualizers/rerun/rerun_visualizer.py index 7e6f9a00331..8b9eea88081 100644 --- a/source/isaaclab_visualizers/isaaclab_visualizers/rerun/rerun_visualizer.py +++ b/source/isaaclab_visualizers/isaaclab_visualizers/rerun/rerun_visualizer.py @@ -8,6 +8,8 @@ from __future__ import annotations import atexit +import contextlib +import inspect import logging import socket import webbrowser @@ -26,7 +28,7 @@ from .rerun_visualizer_cfg import RerunVisualizerCfg if TYPE_CHECKING: - from isaaclab.scene.scene_data_provider import SceneDataProvider + from isaaclab.scene_data import SceneDataProvider logger = logging.getLogger(__name__) @@ -85,15 +87,32 @@ def _open_rerun_web_viewer(host: str, web_port: int, connect_to: str) -> None: def _rerun_web_viewer_url(host: str, web_port: int, connect_to: str) -> str: """Return rerun web UI URL with prefilled endpoint.""" - return f"http://{host}:{int(web_port)}/?url={quote(connect_to, safe='')}" + # Keep the nested URL readable while still encoding '+' in the rerun+http scheme. + return f"http://{host}:{int(web_port)}/?url={quote(connect_to, safe=':/')}" class NewtonViewerRerun(ViewerRerun): """Wrapper around Newton's ViewerRerun with rendering pause controls.""" - def __init__(self, *args, **kwargs): + def __init__(self, *args, open_browser: bool = False, **kwargs): """Initialize viewer wrapper and Isaac Lab pause state.""" - super().__init__(*args, **kwargs) + if open_browser: + super().__init__(*args, **kwargs) + else: + original_serve_web_viewer = rr.serve_web_viewer + + # Rerun Viewer launches a browser automatically, so here we suppress that behavior + def _serve_web_viewer_without_browser(*serve_args, **serve_kwargs): + with contextlib.suppress(TypeError, ValueError): + supports_open_browser = "open_browser" in inspect.signature(original_serve_web_viewer).parameters + if supports_open_browser: + serve_kwargs.setdefault("open_browser", False) + return original_serve_web_viewer(*serve_args, **serve_kwargs) + + with contextlib.ExitStack() as stack: + rr.serve_web_viewer = _serve_web_viewer_without_browser + stack.callback(setattr, rr, "serve_web_viewer", original_serve_web_viewer) + super().__init__(*args, **kwargs) self._paused_rendering = False def is_rendering_paused(self) -> bool: @@ -132,7 +151,6 @@ def __init__(self, cfg: RerunVisualizerCfg): self._step_counter = 0 self._model = None self._state = None - self._scene_data_provider = None self._last_camera_pose: tuple[tuple[float, float, float], tuple[float, float, float]] | None = None self._resolved_visible_env_ids: list[int] | None = None @@ -146,14 +164,12 @@ def initialize(self, scene_data_provider: SceneDataProvider) -> None: if self._is_initialized: return - if scene_data_provider is None: - raise RuntimeError("Rerun visualizer requires a scene_data_provider.") - self._scene_data_provider = scene_data_provider + scene_data_provider = self._set_scene_data_provider(scene_data_provider) num_envs = scene_data_provider.num_envs self._env_ids = self._compute_visualized_env_ids() self._model = NewtonManager.get_model() - self._state = NewtonManager.get_state() + self._state = NewtonManager.get_state(self._scene_data_provider) grpc_port = int(self.cfg.grpc_port) web_port = int(self.cfg.web_port) @@ -177,11 +193,14 @@ def initialize(self, scene_data_provider: SceneDataProvider) -> None: keep_historical_data=self.cfg.keep_historical_data, keep_scalar_history=self.cfg.keep_scalar_history, record_to_rrd=self.cfg.record_to_rrd, + open_browser=self.cfg.open_browser, ) if start_server_in_viewer: rerun_address = getattr(self._viewer, "_grpc_server_uri", rerun_address) viewer_host = _normalize_host(bind_address) viewer_url = _rerun_web_viewer_url(viewer_host, web_port, rerun_address) + print() + self._log_viewer_url("RerunVisualizer", viewer_url) if self.cfg.open_browser and not start_server_in_viewer: _open_rerun_web_viewer(viewer_host, web_port, rerun_address) self._viewer.set_model(self._model) @@ -209,10 +228,9 @@ def initialize(self, scene_data_provider: SceneDataProvider) -> None: rows=[ ("eye", self.cfg.eye), ("lookat", self.cfg.lookat), - ("cam_source", self.cfg.cam_source), + ("focal_length", f"{self.cfg.focal_length} (not applied: Rerun EyeControls3D has no FOV field)"), ("num_visualized_envs", num_visualized_envs), ("endpoint", f"http://{viewer_host}:{web_port}"), - ("viewer_url", viewer_url), ("bind_address", bind_address), ("grpc_port", grpc_port), ("web_port", web_port), @@ -238,10 +256,7 @@ def step(self, dt: float) -> None: self._sim_time += dt self._step_counter += 1 - if self.cfg.cam_source == "prim_path": - self._update_camera_from_usd_path() - - self._state = NewtonManager.get_state() + self._state = NewtonManager.get_state(self._scene_data_provider) num_envs = NewtonManager.get_num_envs() if not self._viewer.is_paused(): @@ -291,15 +306,7 @@ def is_running(self) -> bool: return self._viewer.is_running() def _resolve_initial_camera_pose(self) -> tuple[tuple[float, float, float], tuple[float, float, float]]: - """Resolve initial camera pose from config or USD camera path.""" - if self.cfg.cam_source == "prim_path": - pose = self._resolve_camera_pose_from_usd_path(self.cfg.cam_prim_path) - if pose is not None: - return pose - raise RuntimeError( - "[RerunVisualizer] cam_source='prim_path' requires a resolvable camera prim path, " - f"but no camera pose was found for '{self.cfg.cam_prim_path}'." - ) + """Resolve initial camera pose from config.""" return self._resolve_cfg_camera_pose("RerunVisualizer") def _apply_camera_pose(self, pose: tuple[tuple[float, float, float], tuple[float, float, float]]) -> None: @@ -326,15 +333,6 @@ def _apply_camera_pose(self, pose: tuple[tuple[float, float, float], tuple[float ) self._last_camera_pose = (cam_pos, cam_target) - def _update_camera_from_usd_path(self) -> None: - """Refresh camera pose from configured USD camera path when it changes.""" - pose = self._resolve_camera_pose_from_usd_path(self.cfg.cam_prim_path) - if pose is None: - return - if self._last_camera_pose == pose: - return - self._apply_camera_pose(pose) - def supports_markers(self) -> bool: """Rerun backend supports Isaac Lab markers through Newton viewer primitives.""" return bool(self.cfg.enable_markers) diff --git a/source/isaaclab_visualizers/isaaclab_visualizers/rerun/rerun_visualizer_cfg.py b/source/isaaclab_visualizers/isaaclab_visualizers/rerun/rerun_visualizer_cfg.py index dad671bf57f..c3229a1916d 100644 --- a/source/isaaclab_visualizers/isaaclab_visualizers/rerun/rerun_visualizer_cfg.py +++ b/source/isaaclab_visualizers/isaaclab_visualizers/rerun/rerun_visualizer_cfg.py @@ -22,7 +22,7 @@ class RerunVisualizerCfg(VisualizerCfg): """Application identifier shown in viewer title.""" web_port: int = 9090 - """Port of the local rerun web viewer which is launched in the browser.""" + """Port of the local rerun web viewer whose URL is logged during initialization.""" grpc_port: int = 9876 """Port of the rerun gRPC server (used when serving web viewer externally).""" @@ -36,8 +36,11 @@ class RerunVisualizerCfg(VisualizerCfg): - Local browser links normalize common loopback/wildcard hosts to ``127.0.0.1``. """ - open_browser: bool = True - """Whether to attempt opening the rerun web viewer URL in a browser.""" + open_browser: bool = False + """Whether to attempt opening the rerun web viewer URL in a browser. + + The viewer URL is always logged during initialization. Set this to ``True`` to auto-launch it. + """ keep_historical_data: bool = False """Keep transform history for time scrubbing (False = constant memory for training).""" diff --git a/source/isaaclab_visualizers/isaaclab_visualizers/viser/viser_visualizer.py b/source/isaaclab_visualizers/isaaclab_visualizers/viser/viser_visualizer.py index b3569b04dbf..5bd9a831adc 100644 --- a/source/isaaclab_visualizers/isaaclab_visualizers/viser/viser_visualizer.py +++ b/source/isaaclab_visualizers/isaaclab_visualizers/viser/viser_visualizer.py @@ -10,6 +10,7 @@ import contextlib import io import logging +import math import os import webbrowser from pathlib import Path @@ -27,7 +28,7 @@ logger = logging.getLogger(__name__) if TYPE_CHECKING: - from isaaclab.scene.scene_data_provider import SceneDataProvider + from isaaclab.scene_data import SceneDataProvider def _disable_viser_runtime_client_rebuild_if_bundled() -> None: @@ -46,26 +47,8 @@ def _disable_viser_runtime_client_rebuild_if_bundled() -> None: client_autobuild.ensure_client_is_built = lambda: None -@contextlib.contextmanager -def _suppress_viser_startup_logs(enabled: bool): - """Temporarily quiet noisy viser/websockets startup output.""" - if not enabled: - yield - return - - websockets_logger = logging.getLogger("websockets.server") - previous_level = websockets_logger.level - websockets_logger.setLevel(logging.WARNING) - try: - with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): - yield - finally: - websockets_logger.setLevel(previous_level) - - -def _open_viser_web_viewer(port: int) -> None: - """Open the local viser web UI in a browser.""" - url = _viser_web_viewer_url(port) +def _open_viser_web_viewer(url: str) -> None: + """Open the Viser web UI in a browser.""" try: if not webbrowser.open_new_tab(url): logger.info("[ViserVisualizer] Could not auto-open browser tab. Open manually: %s", url) @@ -73,9 +56,9 @@ def _open_viser_web_viewer(port: int) -> None: logger.info("[ViserVisualizer] Could not auto-open browser tab. Open manually: %s", url) -def _viser_web_viewer_url(port: int) -> str: - """Return local viser web UI URL.""" - return f"http://localhost:{int(port)}" +def _viser_web_viewer_url(port: int, display_address: str) -> str: + """Return Viser web UI URL for display to users.""" + return f"http://{display_address}:{int(port)}" class NewtonViewerViser(ViewerViser): @@ -84,6 +67,7 @@ class NewtonViewerViser(ViewerViser): def __init__( self, port: int = 8080, + bind_address: str = "0.0.0.0", label: str | None = None, verbose: bool = True, share: bool = False, @@ -94,6 +78,7 @@ def __init__( Args: port: HTTP port for viser server. + bind_address: Host/interface for the Viser server to bind. label: Optional viewer label. verbose: Whether to keep verbose startup output enabled. share: Whether to enable sharing/tunneling. @@ -101,15 +86,34 @@ def __init__( metadata: Optional metadata attached to the viewer. """ _disable_viser_runtime_client_rebuild_if_bundled() - super().__init__( - port=port, - label=label, - verbose=verbose, - share=share, - record_to_viser=record_to_viser, - ) + viser = self._get_viser() + original_viser_server = viser.ViserServer + + def _viser_server_with_bind_address(*args, **kwargs): + kwargs["host"] = bind_address + kwargs["verbose"] = verbose + return original_viser_server(*args, **kwargs) + + with contextlib.ExitStack() as stack: + viser.ViserServer = _viser_server_with_bind_address + stack.callback(setattr, viser, "ViserServer", original_viser_server) + if not verbose: + stack.enter_context(contextlib.redirect_stdout(io.StringIO())) + stack.enter_context(contextlib.redirect_stderr(io.StringIO())) + super().__init__( + port=port, + label=label, + verbose=verbose, + share=share, + record_to_viser=record_to_viser, + ) self._metadata = metadata or {} + @property + def share_url(self) -> str | None: + """Return the public share URL created by Viser, if any.""" + return self._share_url + class ViserVisualizer(BaseVisualizer): """Viser web-based visualizer backed by Newton's ViewerViser.""" @@ -126,7 +130,6 @@ def __init__(self, cfg: ViserVisualizerCfg): self._model: Any | None = None self._state = None self._sim_time = 0.0 - self._scene_data_provider = None self._active_record_path: str | None = None self._last_camera_pose: tuple[tuple[float, float, float], tuple[float, float, float]] | None = None self._pending_camera_pose: tuple[tuple[float, float, float], tuple[float, float, float]] | None = None @@ -144,15 +147,13 @@ def initialize(self, scene_data_provider: SceneDataProvider) -> None: if self._is_initialized: logger.debug("[ViserVisualizer] initialize() called while already initialized.") return - if scene_data_provider is None: - raise RuntimeError("Viser visualizer requires a scene_data_provider.") - self._scene_data_provider = scene_data_provider + scene_data_provider = self._set_scene_data_provider(scene_data_provider) num_envs = scene_data_provider.num_envs metadata = {"num_envs": num_envs} self._env_ids = self._compute_visualized_env_ids() self._model = NewtonManager.get_model() - self._state = NewtonManager.get_state() + self._state = NewtonManager.get_state(self._scene_data_provider) self._active_record_path = self.cfg.record_to_viser self._create_viewer(record_to_viser=self.cfg.record_to_viser, metadata=metadata) @@ -160,17 +161,17 @@ def initialize(self, scene_data_provider: SceneDataProvider) -> None: num_visualized_envs = ( len(self._resolved_visible_env_ids) if self._resolved_visible_env_ids is not None else num_envs ) - viewer_url = _viser_web_viewer_url(self.cfg.port) self._log_initialization_table( logger=logger, title="ViserVisualizer Configuration", rows=[ ("eye", self.cfg.eye), ("lookat", self.cfg.lookat), - ("cam_source", self.cfg.cam_source), + ("focal_length", self.cfg.focal_length), ("num_visualized_envs", num_visualized_envs), + ("bind_address", self.cfg.bind_address), + ("display_address", self.cfg.display_address), ("port", self.cfg.port), - ("viewer_url", viewer_url), ("record_to_viser", self.cfg.record_to_viser or ""), ], ) @@ -187,11 +188,9 @@ def step(self, dt: float) -> None: if not self._is_initialized or self._viewer is None or self._scene_data_provider is None: return - if self.cfg.cam_source == "prim_path": - self._update_camera_from_usd_path() self._apply_pending_camera_pose() - self._state = NewtonManager.get_state() + self._state = NewtonManager.get_state(self._scene_data_provider) num_envs = NewtonManager.get_num_envs() self._sim_time += dt @@ -266,14 +265,21 @@ def _create_viewer(self, record_to_viser: str | None, metadata: dict | None = No if self._model is None: raise RuntimeError("Viser visualizer requires a Newton model.") - with _suppress_viser_startup_logs(enabled=not self.cfg.verbose): - self._viewer = NewtonViewerViser( - port=self.cfg.port, - label=self.cfg.label, - verbose=self.cfg.verbose, - share=self.cfg.share, - record_to_viser=record_to_viser, - metadata=metadata or {}, + self._viewer = NewtonViewerViser( + port=self.cfg.port, + bind_address=self.cfg.bind_address, + label=self.cfg.label, + verbose=False, + share=self.cfg.share, + record_to_viser=record_to_viser, + metadata=metadata or {}, + ) + viewer_url = self._viewer.share_url or _viser_web_viewer_url(self.cfg.port, self.cfg.display_address) + if self.cfg.verbose: + print() + self._log_viewer_url( + "ViserVisualizer", + viewer_url, ) num_envs = int((metadata or {}).get("num_envs", 0)) self._viewer.set_model(self._model) @@ -286,7 +292,7 @@ def _create_viewer(self, record_to_viser: str | None, metadata: dict | None = No # Preserve simulation world positions (env_spacing) rather than adding viewer-side offsets. self._viewer.set_world_offsets((0.0, 0.0, 0.0)) if self.cfg.open_browser: - _open_viser_web_viewer(self.cfg.port) + _open_viser_web_viewer(viewer_url) initial_pose = self._resolve_initial_camera_pose() self._set_viser_camera_view(initial_pose) self._sim_time = 0.0 @@ -305,15 +311,7 @@ def _close_viewer(self, finalize_viser: bool = False) -> None: self._viewer = None def _resolve_initial_camera_pose(self) -> tuple[tuple[float, float, float], tuple[float, float, float]]: - """Resolve initial camera pose from config or USD camera path.""" - if self.cfg.cam_source == "prim_path": - pose = self._resolve_camera_pose_from_usd_path(self.cfg.cam_prim_path) - if pose is not None: - return pose - raise RuntimeError( - "[ViserVisualizer] cam_source='prim_path' requires a resolvable camera prim path, " - f"but no camera pose was found for '{self.cfg.cam_prim_path}'." - ) + """Resolve initial camera pose from config.""" return self._resolve_cfg_camera_pose("ViserVisualizer") def _try_apply_viser_camera_view(self, pose: tuple[tuple[float, float, float], tuple[float, float, float]]) -> bool: @@ -336,12 +334,16 @@ def _try_apply_viser_camera_view(self, pose: tuple[tuple[float, float, float], t client_iterable = clients.values() if isinstance(clients, dict) else clients cam_pos, cam_target = pose + fov_radians = math.radians(self._focal_length_to_vertical_fov_degrees()) applied = False for client in client_iterable: camera = getattr(client, "camera", None) if camera is None: continue try: + if hasattr(camera, "fov"): + camera.fov = fov_radians + applied = True if hasattr(camera, "position"): camera.position = cam_pos applied = True @@ -367,12 +369,3 @@ def _apply_pending_camera_pose(self) -> None: if self._try_apply_viser_camera_view(self._pending_camera_pose): self._last_camera_pose = self._pending_camera_pose self._pending_camera_pose = None - - def _update_camera_from_usd_path(self) -> None: - """Refresh camera pose from configured USD camera path when it changes.""" - pose = self._resolve_camera_pose_from_usd_path(self.cfg.cam_prim_path) - if pose is None: - return - if self._last_camera_pose == pose or self._pending_camera_pose == pose: - return - self._set_viser_camera_view(pose) diff --git a/source/isaaclab_visualizers/isaaclab_visualizers/viser/viser_visualizer_cfg.py b/source/isaaclab_visualizers/isaaclab_visualizers/viser/viser_visualizer_cfg.py index 68ab3116b45..b039924d392 100644 --- a/source/isaaclab_visualizers/isaaclab_visualizers/viser/viser_visualizer_cfg.py +++ b/source/isaaclab_visualizers/isaaclab_visualizers/viser/viser_visualizer_cfg.py @@ -21,8 +21,23 @@ class ViserVisualizerCfg(VisualizerCfg): port: int = 8080 """Port of the local viser web server.""" - open_browser: bool = True - """Whether to attempt opening the viser web viewer URL in a browser.""" + bind_address: str = "0.0.0.0" + """Host/interface for the Viser server to bind. + + Use ``"0.0.0.0"`` to listen on all interfaces for remote access. + """ + + display_address: str = "localhost" + """Host name or IP address shown in the printed browser URL. + + For remote access, set this to the hostname/IP reachable from your browser. + """ + + open_browser: bool = False + """Whether to attempt opening the viser web viewer URL in a browser. + + The viewer URL is always logged during initialization. Set this to ``True`` to auto-launch it. + """ label: str | None = "Isaac Lab Simulation" """Optional label shown in the viewer page title.""" diff --git a/source/isaaclab_visualizers/test/test_visualizer_cartpole_integration.py b/source/isaaclab_visualizers/test/test_visualizer_cartpole_integration.py index 79bad6ea145..bcdeac5e241 100644 --- a/source/isaaclab_visualizers/test/test_visualizer_cartpole_integration.py +++ b/source/isaaclab_visualizers/test/test_visualizer_cartpole_integration.py @@ -522,6 +522,7 @@ def test_cartpole_newton_visualizer_tiled_camera_rgb_non_black( @pytest.mark.isaacsim_ci +@pytest.mark.skip(reason="ViewerGL frame motion is flaky on the current pinned Isaac Sim CI image.") @pytest.mark.parametrize("backend_kind", ["physx", "newton"]) def test_cartpole_newton_visualizer_viewergl_rgb_motion(backend_kind: str, caplog: pytest.LogCaptureFixture) -> None: """Newton GL (``ViewerGL.get_frame``): full motion steps, last frame non-black; early vs late differ; logs."""