seg_semantic of camera doesn't work when there is a swimingpool filled with water in the scene

[SresserS]

I can't get the segmentation at this scene. However , when I remove swimingpool.states[Filled].set_value(water, True) , the segmentation works

[SresserS]

[Warning] [omnigibson] Creating an instancer that uses isosurface /World/water/waterInstancer0/instancer. The rendering of these particles will have a delay of one timestep.

[Error] [omni.syntheticdata.plugin] SdStageSemanticInstanceMapping : number of semantics is greater than the maximum number of uids
2024-09-08 07:14:17 [16,597ms] [Error] [omni.syntheticdata.plugin] OgnSdStageSemanticInstanceMapping missing filteredLabelMap semLabelMap in the history.
2024-09-08 07:14:17 [16,597ms] [Warning] [omni.replicator.core.plugin] InstanceSegmentation : missing RenderVar InstanceMapSD
2024-09-08 07:14:17 [16,615ms] [Warning] [omni.syntheticdata.plugin] OgnSdInstanceMapping missing valid input renderVar InstanceMappingInfoSDhost
2024-09-08 07:14:17 [16,620ms] [Warning] [omni.syntheticdata.plugin] OgnSdInstanceMapping missing valid input renderVar InstanceMappingInfoSDhost
2024-09-08 07:14:17 [16,620ms] [Warning] [omni.syntheticdata.plugin] SdRenderVarToRawArray missing valid input renderVar InstanceSegmentationReductionSD
2024-09-08 07:14:17 [16,633ms] [Error] [omni.syntheticdata.plugin] OgnSdStageSemanticInstanceMapping missing filteredLabelMap semLabelMap in the history.
2024-09-08 07:14:17 [16,696ms] [Error] [omni.syntheticdata.plugin] OgnSdStageSemanticInstanceMapping missing filteredLabelMap semLabelMap in the history.

File "/home/nvidia/simulators/OmniGibson/omnigibson/sensors/sensor_base.py", line 76, in get_obs
obs, info = self._get_obs()
File "/home/nvidia/simulators/OmniGibson/omnigibson/sensors/vision_sensor.py", line 274, in _get_obs
obs[modality], info[modality] = self._remap_semantic_segmentation(obs[modality], id_to_labels)
File "/home/nvidia/simulators/OmniGibson/omnigibson/sensors/vision_sensor.py", line 316, in _remap_semantic_segmentation
return VisionSensor.SEMANTIC_REMAPPER.remap(replicator_mapping, semantic_class_id_to_name(), img)
File "/home/nvidia/simulators/OmniGibson/omnigibson/utils/vision_utils.py", line 94, in remap
image_max_key = np.max(image)
File "<array_function internals>", line 180, in amax
File "/home/nvidia/anaconda3/envs/omnigibson/lib/python3.10/site-packages/numpy/core/fromnumeric.py", line 2793, in amax
return _wrapreduction(a, np.maximum, 'max', axis, None, out,
File "/home/nvidia/anaconda3/envs/omnigibson/lib/python3.10/site-packages/numpy/core/fromnumeric.py", line 86, in _wrapreduction
return ufunc.reduce(obj, axis, dtype, out, **passkwargs)
ValueError: zero-size array to reduction operation maximum which has no identity

[SresserS]

2024-09-08 07:14:19 [18,420ms] [Warning] [omni.graph.core.plugin] Could not find category 'Replicator:Annotators' for removal
2024-09-08 07:14:19 [18,420ms] [Warning] [omni.graph.core.plugin] Could not find category 'Replicator:Annotators' for removal
2024-09-08 07:14:19 [18,420ms] [Warning] [omni.graph.core.plugin] Could not find category 'Replicator:Annotators' for removal

[hang-yin]

Hi @SresserS , thanks for reaching out! Do you mind sending the specific scene you are loading so that I can reproduce this on my end?

[SresserS]

Sorry to have kept you waiting. @hang-yin
The following content is modified based on issue #700, adding a sink.

`import numpy as np
import cv2

import os
import sys

sys.path.append(r'/git/OmniGibson')

os.environ['KMP_DUPLICATE_LIB_OK']='True'

import omnigibson as og
from omnigibson.object_states import Covered
from omnigibson.objects import DatasetObject
from omnigibson.macros import gm, macros
from omnigibson.systems import get_system
from omnigibson.utils.usd_utils import create_joint
from omnigibson.utils.ui_utils import choose_from_options
from omnigibson.utils.constants import ParticleModifyMethod

Set macros for this example

macros.object_states.particle_modifier.VISUAL_PARTICLES_REMOVAL_LIMIT = 1000
macros.object_states.particle_modifier.PHYSICAL_PARTICLES_REMOVAL_LIMIT = 8000
macros.object_states.particle_modifier.MAX_VISUAL_PARTICLES_APPLIED_PER_STEP = 4
macros.object_states.particle_modifier.MAX_PHYSICAL_PARTICLES_APPLIED_PER_STEP = 40
macros.object_states.covered.MAX_VISUAL_PARTICLES = 300

Make sure object states and GPU dynamics are enabled (GPU dynamics needed for fluids)

gm.ENABLE_OBJECT_STATES = True
gm.USE_GPU_DYNAMICS = True
gm.ENABLE_HQ_RENDERING = True

def main(random_selection=False, headless=False, short_exec=False):

modifier_type = "particleApplier"
modification_metalink = {
    "particleApplier": "particleapplier_link",
    "particleRemover": "particleremover_link",
}
particle_type = "water" #antifreeze

modification_method = {
    "Adjacency": ParticleModifyMethod.ADJACENCY,
    "Projection": ParticleModifyMethod.PROJECTION,
}

projection_mesh_params = {
    "Adjacency": None,
    "Projection": {
        "type": "Cone",
        "extents": np.array([0.1875, 0.1875, 0.375]),
    },
}

method_type = "Projection"

abilities = {
    modifier_type: {
        "method": modification_method[method_type],
        "conditions": {
            particle_type: [],
        },
        "projection_mesh_params": projection_mesh_params[method_type],
    }
}

table_cfg = dict(
    type="DatasetObject",
    name="table",
    category="breakfast_table",
    model="kwmfdg",
    bounding_box=[3.402, 1.745, 1.175],
    position=[0, 0, 0.98],
)
bathtub_cfg = dict(
    type="DatasetObject",
    name="hot_tub",
    category="hot_tub",
    model="nuzkuf",
    abilities={
        "fillable":{},
    },
    scale=0.8 #(-0.9~0.9,-0.9~0.9, 0.05)
) 

cfg = {
    "scene": {
        "type": "Scene",
    },
    "objects": [table_cfg,bathtub_cfg],
}

env = og.Environment(configs=cfg)
og.sim.stop()

cam = og.sim.viewer_camera
cam.set_position_orientation(
    position=np.array([-1.61340969, -1.79803028,  2.53167412]),
    orientation=np.array([ 0.46291845, -0.12381886, -0.22679218,  0.84790371]),
)
cam.add_modality('seg_semantic')
cam.add_modality('seg_instance')

sink = env.scene.object_registry("name", "hot_tub")
water=get_system("water")
og.log.info(sink.states.keys())
from omnigibson import object_states
sink.states[object_states.Filled].set_value(water,True)

# If we're using a projection volume, we manually add in the required metalink required in order to use the volume
modifier = DatasetObject(
    name="modifier",
    category="dishtowel",
    model="dtfspn",
    bounding_box=[0.34245, 0.46798, 0.07],
    visual_only=method_type == "Projection",  # Non-fluid adjacency requires the object to have collision geoms active
    abilities=abilities,
)

modifier_root_link_path = f"{modifier.prim_path}/base_link"
modifier._prim = modifier._load()
if method_type == "Projection":
    metalink_path = f"{modifier.prim_path}/{modification_metalink[modifier_type]}"
    og.sim.stage.DefinePrim(metalink_path, "Xform")
    create_joint(
        prim_path=f"{modifier_root_link_path}/{modification_metalink[modifier_type]}_joint",
        body0=modifier_root_link_path,
        body1=metalink_path,
        joint_type="FixedJoint",
        enabled=True,
    )

modifier._post_load()
modifier._loaded = True
og.sim.import_object(modifier)
modifier.set_position(np.array([0, 0, 5.0]))

og.sim.play()
for _ in range(25):
    env.step(np.array([]))

og.sim.enable_viewer_camera_teleoperation()

#method_type == "Projection":
z = 1.85

modifier.keep_still()
modifier.set_position_orientation(
    position=np.array([0, 0.3, z]),
    orientation=np.array([0, 0, 0, 1.0]),
)

deltas = [
    [5, np.array([0, 0, 0])],
]
for t, delta in deltas:
    for i in range(t):
        modifier.set_position(modifier.get_position() + delta)
        env.step(np.array([]))

# og.sim.remove_object(modifier)
for i in range(10):
    og.sim.render()
print(np.array(cam.get_obs()[0]["seg_instance"]))
print(np.unique(np.array(cam.get_obs()[0]["seg_instance"])))
while True:
    env.step(np.array([]))
    og.sim.step()
    og.sim.render()
    cv2.imshow('seg', np.array(cam.get_obs()[0]["seg_instance"]*80, dtype=np.uint8))
    cv2.waitKey(1)

env.close()

if name == "main":
main()
`

[hang-yin]

Hi @SresserS , sorry for taking so long to get back to you! It seems to me that omniverse replicator segmentation annotators can not handle a huge amount of particles. One thing that you could try is that in our omnigibson/systems/micro_particle_systems.py:FluidSystem, you can try to make the particle_radius property larger so that we have less particles instantiated into the hot tub.

[SresserS]

Thanks! That really helps a lot! 😀